dillard
dfe1fe9aad
A file whose data cannot be read or decrypted is left out and listed, and the command exits with code 2. When the database cannot be restored (its dump is damaged, or it does not fit this installation), backup:restore:full leaves the fresh database as it is and puts the data directory and config of the backup in ngb-salvage-<snapshot> in the data directory. Without --salvage a restore of damaged data stops as before and points at the option.
1286 lines
50 KiB
PHP
1286 lines
50 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
namespace OCA\NgBackup\Repository;
|
|
|
|
use OCA\NgBackup\Backend\BackendException;
|
|
use OCA\NgBackup\Backend\IBackend;
|
|
use OCA\NgBackup\Backend\IReadsAhead;
|
|
use OCA\NgBackup\Crypto\KeyRing;
|
|
use OCA\NgBackup\Crypto\StreamCipher;
|
|
|
|
/**
|
|
* Content-addressed, encrypted, deduplicating repository.
|
|
*
|
|
* Layout on the backend:
|
|
* config plaintext JSON: format version, repository id, wrapped master key
|
|
* packs/xx/<packId> encrypted blobs + encrypted pack header
|
|
* index/<packId> encrypted list of the blobs in that pack
|
|
* snapshots/<snapId> encrypted JSON lines: one meta line, then one line per file
|
|
*
|
|
* Files are read as streams and cut into blobs of BLOB_SIZE; only one blob and one pack are in
|
|
* memory at a time. A blob that already exists (same keyed hash) is not stored again.
|
|
*/
|
|
final class Repository {
|
|
public const FORMAT = 1;
|
|
public const BLOB_SIZE = 4 * 1048576;
|
|
private const FLAG_DEFLATE = 1;
|
|
/** Blobs from this size on get a quick compression test before the full compression. */
|
|
private const DEFLATE_PROBE_FROM = 256 * 1024;
|
|
/** Bytes of each of the three slices (start, middle, end) in that test. */
|
|
private const DEFLATE_PROBE_SLICE = 32 * 1024;
|
|
/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
|
|
private const VERIFY_READ_ATTEMPTS = 3;
|
|
/** Largest read that loadBlobs() makes of blobs that follow each other in a pack. */
|
|
private const MERGED_READ_MAX = 16 * 1048576;
|
|
|
|
private StreamCipher $cipher;
|
|
private BlobIndex $index;
|
|
private ?PackWriter $runPacks = null;
|
|
private ?int $maxBytes = null;
|
|
private Catalog $catalog;
|
|
private PackReader $packReader;
|
|
/** verify() lists the pack folders from this many packs on (see presentPacks()); tests lower it. */
|
|
public int $listPacksAbove = 256;
|
|
|
|
private function __construct(
|
|
private IBackend $backend,
|
|
private KeyRing $keys,
|
|
private string $repositoryId,
|
|
) {
|
|
$this->cipher = new StreamCipher($keys);
|
|
$this->packReader = new PackReader($backend);
|
|
$this->backend = new PackReaderGuard($backend, $this->packReader);
|
|
$this->index = new BlobIndex($this->backend, $this->cipher);
|
|
}
|
|
|
|
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
|
$keys = KeyRing::generate();
|
|
return self::initWithKey($backend, $keys, $keys->wrap($passphrase));
|
|
}
|
|
|
|
/**
|
|
* Create a repository with an existing master key. $wrappedKey is the passphrase-wrapped
|
|
* copy (KeyRing::wrap) stored in the repository config, so the repository can be opened
|
|
* with the passphrase from the recovery kit even when the server is gone.
|
|
*/
|
|
public static function initWithKey(IBackend $backend, KeyRing $keys, array $wrappedKey, ?ICatalogAnchor $anchor = null): self {
|
|
if ($backend->exists('config')) {
|
|
throw new RepositoryException('A repository already exists at this location');
|
|
}
|
|
// No config, but backup data: the config is missing (or could not be seen just now, a
|
|
// connection that failed). A new repository here would write a config over the one that
|
|
// opens this data; refuse instead.
|
|
foreach (['index', 'snapshots', 'trash/snapshots', 'catalog'] as $dir) {
|
|
if ($backend->listFiles($dir) !== []) {
|
|
throw new RepositoryException("This location holds backup data ($dir/) but no repository config could be read: not creating a new repository over it. Check the connection and the folder; a missing config can be restored from the recovery kit.");
|
|
}
|
|
}
|
|
$id = bin2hex(random_bytes(16));
|
|
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
|
|
'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))];
|
|
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
|
$repo = new self($backend, $keys, $id);
|
|
$repo->loadCatalog($anchor);
|
|
return $repo;
|
|
}
|
|
|
|
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
|
|
/** @param ?callable(int, int): void $progress index files downloaded so far, and in all (see BlobIndex::load()) */
|
|
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null, ?callable $progress = null): self {
|
|
$config = self::readConfig($backend);
|
|
try {
|
|
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
|
|
} catch (\OCA\NgBackup\Crypto\CryptoException) {
|
|
$ok = false;
|
|
}
|
|
if (!$ok) {
|
|
throw new RepositoryException('This repository was created with a different key');
|
|
}
|
|
$repo = new self($backend, $keys, $config['id']);
|
|
if ($cache !== null) {
|
|
$repo->index->setCache($cache, $config['id']);
|
|
}
|
|
$repo->index->load(false, $progress);
|
|
$repo->loadCatalog($anchor);
|
|
return $repo;
|
|
}
|
|
|
|
private function loadCatalog(?ICatalogAnchor $anchor): void {
|
|
$this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor);
|
|
$this->catalog->load(
|
|
fn () => array_map('basename', $this->backend->listFiles('snapshots')),
|
|
fn () => array_map('basename', $this->backend->listFiles('trash/info')),
|
|
fn () => $this->backend->list('users'));
|
|
}
|
|
|
|
/** Refuse uploads that would make the stored data larger than this (null = no limit). */
|
|
public function setMaxBytes(?int $maxBytes): void {
|
|
$this->maxBytes = $maxBytes;
|
|
}
|
|
|
|
public function storedBytes(): int {
|
|
return $this->index->storedBytes();
|
|
}
|
|
|
|
/** Current catalog generation (for status and tests). */
|
|
public function catalogGeneration(): int {
|
|
return $this->catalog->gen;
|
|
}
|
|
|
|
/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
|
|
public function indexDownloads(): int {
|
|
return $this->index->downloaded;
|
|
}
|
|
|
|
private static function readConfig(IBackend $backend): array {
|
|
try {
|
|
$fh = $backend->get('config');
|
|
} catch (BackendException) {
|
|
throw new RepositoryException('No repository at this location');
|
|
}
|
|
$config = json_decode((string)stream_get_contents($fh), true, 512, JSON_THROW_ON_ERROR);
|
|
fclose($fh);
|
|
if (($config['format'] ?? 0) !== self::FORMAT) {
|
|
throw new RepositoryException('Unsupported repository format');
|
|
}
|
|
return $config;
|
|
}
|
|
|
|
/** Replace the passphrase-wrapped key slots in this repository's config (after a slot change). */
|
|
public function updateWrappedKey(array $wrappedKey): void {
|
|
$config = self::readConfig($this->backend);
|
|
$config['key'] = $wrappedKey;
|
|
self::putString($this->backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
|
}
|
|
|
|
public function id(): string {
|
|
return $this->repositoryId;
|
|
}
|
|
|
|
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
|
$config = self::readConfig($backend);
|
|
$repo = new self($backend, KeyRing::unwrapAny($config['key'], $passphrase), $config['id']);
|
|
$repo->index->load();
|
|
$repo->loadCatalog(null); // disaster recovery on a new server: no anchor yet, trust the newest valid chain
|
|
return $repo;
|
|
}
|
|
|
|
/**
|
|
* Back up a local directory tree. Files whose size and mtime match the parent snapshot are not
|
|
* read again; their blob list is reused.
|
|
*
|
|
* @return array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int}
|
|
*/
|
|
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
|
$source = rtrim($source, '/');
|
|
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
|
$packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
|
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
|
|
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$snapshotId = bin2hex(random_bytes(16));
|
|
fwrite($tree, json_encode(['snapshot' => $snapshotId, 'time' => gmdate('c'), 'parent' => $parent, 'label' => $label, 'source' => $source], JSON_THROW_ON_ERROR) . "\n");
|
|
|
|
foreach ($this->walk($source) as $relative => $info) {
|
|
$stats['files']++;
|
|
$prev = $previous[$relative] ?? null;
|
|
if ($prev !== null && $prev['s'] === $info->getSize() && $prev['m'] === $info->getMTime()) {
|
|
$blobs = $prev['b'];
|
|
$stats['reused']++;
|
|
} else {
|
|
$blobs = $this->readStable($info->getPathname(), $packs, $stats);
|
|
$stats['read']++;
|
|
}
|
|
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
|
|
}
|
|
$packs->flush();
|
|
|
|
// The snapshot is written last: it only references blobs whose packs are already stored.
|
|
rewind($tree);
|
|
$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
|
|
rewind($encrypted);
|
|
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
|
fclose($tree);
|
|
$this->catalog->snapshots[$snapshotId] = true;
|
|
$this->catalog->write();
|
|
|
|
return ['snapshot' => $snapshotId] + $stats;
|
|
}
|
|
|
|
/**
|
|
* Restore a snapshot (or only paths starting with $prefix) into a local directory. A $prefix
|
|
* "dir/file/" also restores the file "dir/file" itself.
|
|
*
|
|
* @param bool $stripPrefix write "$target/<path without $prefix>" instead of
|
|
* "$target/<full logical path>" (disaster recovery: restoring e.g. "data/" or
|
|
* "config/" directly into the real data/config directory, not a subdirectory
|
|
* named after the prefix). Ignored when $prefix is '' (nothing to strip).
|
|
* @param array<string, string>|null $skipped salvage: an array to collect the files that could
|
|
* not be read or decrypted (path => reason), which are then left out instead of
|
|
* stopping the restore; null (default) stops at the first such file
|
|
*/
|
|
public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null): int {
|
|
if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) {
|
|
throw new RepositoryException('Invalid path prefix');
|
|
}
|
|
$plan = $this->readAhead($snapshotId, $prefix, $target);
|
|
try {
|
|
return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan, $skipped);
|
|
} finally {
|
|
$plan?->close();
|
|
if ($plan !== null) {
|
|
@rmdir($plan->spool);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start reading the packs of a restore ahead, when the location can (IReadsAhead): a first
|
|
* pass over the snapshot lists the ranges of packs it needs, in order. The spool is a
|
|
* folder in the target, which has room for the restore anyway.
|
|
*/
|
|
private function readAhead(string $snapshotId, string $prefix, string $target): ?ReadAheadPlan {
|
|
if (!$this->backend instanceof IReadsAhead || !$this->backend->canReadAhead()) {
|
|
return null;
|
|
}
|
|
// what a restore that was stopped left behind: empty, its files were unlinked when opened
|
|
$stale = glob(rtrim($target, '/') . '/.ngb-restore-spool-*', GLOB_ONLYDIR);
|
|
foreach ($stale === false ? [] : $stale as $dir) {
|
|
@rmdir($dir);
|
|
}
|
|
$plan = new ReadAheadPlan(rtrim($target, '/') . '/.ngb-restore-spool-' . bin2hex(random_bytes(4)));
|
|
try {
|
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
|
if (!self::underPrefix($entry['p'], $prefix)) {
|
|
continue;
|
|
}
|
|
foreach ($entry['b'] as $id) {
|
|
try {
|
|
[$pack, $offset, $length] = $this->index->get($id);
|
|
} catch (RepositoryException) {
|
|
continue; // the restore reports it when it gets there
|
|
}
|
|
$plan->add('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
|
}
|
|
}
|
|
} catch (\Throwable) {
|
|
return null; // reading ahead is only faster: the restore itself reads the snapshot again and reports what is wrong
|
|
}
|
|
if (count($plan->ranges()) < 2 || (!is_dir($plan->spool) && !@mkdir($plan->spool, 0700, true))) {
|
|
return null;
|
|
}
|
|
try {
|
|
$streams = $this->backend->readAhead($plan->ranges(), $plan->spool);
|
|
} catch (\Throwable) {
|
|
$streams = null; // the usual reads report what is wrong
|
|
}
|
|
if ($streams === null) {
|
|
@rmdir($plan->spool);
|
|
return null;
|
|
}
|
|
$plan->readFrom($streams);
|
|
return $plan;
|
|
}
|
|
|
|
/** @param array<string, string>|null $skipped see restore() */
|
|
private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan, ?array &$skipped): int {
|
|
$count = 0;
|
|
$cleaned = [];
|
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
|
if (!self::underPrefix($entry['p'], $prefix)) {
|
|
continue;
|
|
}
|
|
$relPath = $entry['p'];
|
|
if ($stripPrefix && $prefix !== '') {
|
|
$relPath = $entry['p'] === rtrim($prefix, '/') ? basename($entry['p']) : substr($entry['p'], strlen($prefix));
|
|
}
|
|
$dest = rtrim($target, '/') . '/' . $relPath;
|
|
if (!is_dir(dirname($dest))) {
|
|
mkdir(dirname($dest), 0700, true);
|
|
}
|
|
// Defence in depth: the resolved directory must stay below the target (no symlink escapes).
|
|
$base = realpath($target);
|
|
$dir = realpath(dirname($dest));
|
|
if ($base === false || $dir === false || ($dir !== $base && !str_starts_with($dir, $base . '/')) || is_link($dest)) {
|
|
throw new RepositoryException('Refusing to write outside the restore directory: ' . $entry['p']);
|
|
}
|
|
if (!isset($cleaned[$dir])) {
|
|
self::removeStaleTemporaries($dir);
|
|
$cleaned[$dir] = true;
|
|
}
|
|
// Write next to the destination and rename after all blobs verified: at most one file
|
|
// is ever present twice.
|
|
$tmp = $dest . '.ngb-restore-' . bin2hex(random_bytes(6));
|
|
$out = @fopen($tmp, 'xb'); // fails if anything (including a symlink) already exists there
|
|
if ($out === false) {
|
|
throw new RepositoryException('Cannot create a temporary file for ' . $entry['p']);
|
|
}
|
|
try {
|
|
foreach ($entry['b'] as $id) {
|
|
fwrite($out, $this->restoreBlob($id, $plan));
|
|
if ($heartbeat !== null) {
|
|
$heartbeat();
|
|
}
|
|
}
|
|
fclose($out);
|
|
clearstatcache(true, $tmp);
|
|
if (filesize($tmp) !== $entry['s']) {
|
|
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
|
}
|
|
} catch (RepositoryException|BackendException|\OCA\NgBackup\Crypto\CryptoException $e) {
|
|
// What the location gave back for this file is missing or damaged. Writing to the
|
|
// target (below) is not part of this: a full disk must still stop the restore.
|
|
if (is_resource($out)) {
|
|
fclose($out);
|
|
}
|
|
@unlink($tmp);
|
|
if ($skipped === null) {
|
|
throw $e;
|
|
}
|
|
$skipped[$entry['p']] = $e->getMessage();
|
|
continue;
|
|
}
|
|
if (!rename($tmp, $dest)) {
|
|
@unlink($tmp);
|
|
throw new RepositoryException('Cannot put the restored file in place: ' . $entry['p']);
|
|
}
|
|
if (!touch($dest, $entry['m'])) {
|
|
throw new RepositoryException('Cannot set the modification time of ' . $entry['p']);
|
|
}
|
|
$count++;
|
|
}
|
|
return $count;
|
|
}
|
|
|
|
/** $prefix is "" (everything) or ends with "/"; the path without that "/" is a file to restore too. */
|
|
private static function underPrefix(string $path, string $prefix): bool {
|
|
return $prefix === '' || str_starts_with($path, $prefix) || $path === rtrim($prefix, '/');
|
|
}
|
|
|
|
/**
|
|
* Remove what a restore that was stopped left in a directory this restore writes to:
|
|
* "<name>.ngb-restore-<hex>" from here, ".<name>.ngb-restore-<hex>" from a restore through
|
|
* Nextcloud. A files:scan would otherwise make them show up in Files.
|
|
*/
|
|
public static function removeStaleTemporaries(string $dir): void {
|
|
$names = scandir($dir);
|
|
foreach ($names === false ? [] : $names as $name) {
|
|
$path = $dir . '/' . $name;
|
|
if (self::isRestoreTemporary($name) && is_file($path) && !is_link($path)) {
|
|
@unlink($path);
|
|
}
|
|
}
|
|
}
|
|
|
|
public static function isRestoreTemporary(string $name): bool {
|
|
return preg_match('/.\.ngb-restore-[0-9a-f]{6}(?:[0-9a-f]{6})?$/', $name) === 1;
|
|
}
|
|
|
|
/**
|
|
* Start writing a generated stream (e.g. a table dump) as blobs. Call flushPacks() when the
|
|
* run is done, so the last pack is uploaded before anything references it.
|
|
*/
|
|
public function blobWriter(array &$stats): BlobWriter {
|
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
|
$packs = $this->runPacks;
|
|
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
|
|
return $this->storeBlob($data, $packs, $stats);
|
|
});
|
|
}
|
|
|
|
/** See BlobIndex::holdCache(): no database writes until releaseIndexCache(). */
|
|
public function holdIndexCache(): void {
|
|
$this->index->holdCache();
|
|
}
|
|
|
|
public function releaseIndexCache(): void {
|
|
$this->index->releaseCache();
|
|
}
|
|
|
|
public function flushPacks(): void {
|
|
$this->runPacks?->flush();
|
|
}
|
|
|
|
/**
|
|
* Read a blob stream back as lines (lines may span blob boundaries).
|
|
*
|
|
* @param list<string> $blobIds
|
|
* @return \Generator<string>
|
|
*/
|
|
public function readLines(array $blobIds): \Generator {
|
|
$carry = '';
|
|
foreach ($this->loadBlobs($blobIds) as $blob) {
|
|
$data = $carry . $blob;
|
|
$lines = explode("\n", $data);
|
|
$carry = array_pop($lines);
|
|
yield from $lines;
|
|
}
|
|
if ($carry !== '') {
|
|
yield $carry;
|
|
}
|
|
}
|
|
|
|
/** Store one blob of file data in the current run (deduplicated); returns its id. */
|
|
public function storeData(string $data, array &$stats): string {
|
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
|
return $this->storeBlob($data, $this->runPacks, $stats);
|
|
}
|
|
|
|
/**
|
|
* Write a snapshot from a meta record and the tree lines (JSON, one per file). Must be called
|
|
* after flushPacks(), so the snapshot only references stored blobs.
|
|
*
|
|
* @param iterable<string> $lines
|
|
*/
|
|
public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
|
$this->putSnapshot($snapshotId, $meta, $lines);
|
|
$this->recordSnapshot($snapshotId);
|
|
}
|
|
|
|
/**
|
|
* Store a snapshot object without touching the catalog. Use recordSnapshot() afterwards, from a
|
|
* repository opened inside the catalog lock, so the catalog write builds on the latest generation.
|
|
*
|
|
* @param iterable<string> $lines
|
|
*/
|
|
public function putSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n");
|
|
foreach ($lines as $line) {
|
|
if ($line !== '') {
|
|
fwrite($tree, $line . "\n");
|
|
}
|
|
}
|
|
rewind($tree);
|
|
$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
|
|
fclose($tree);
|
|
rewind($encrypted);
|
|
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
|
}
|
|
|
|
public function recordSnapshot(string $snapshotId): void {
|
|
$this->catalog->snapshots[$snapshotId] = true;
|
|
$this->catalog->write();
|
|
}
|
|
|
|
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
|
public function snapshotEntries(string $snapshotId): array {
|
|
return $this->loadEntries($snapshotId);
|
|
}
|
|
|
|
/**
|
|
* @param list<string> $blobIds
|
|
* @return \Generator<string> verified blob contents, in order
|
|
*/
|
|
public function readBlobs(array $blobIds): \Generator {
|
|
foreach ($blobIds as $id) {
|
|
yield $this->loadBlob($id);
|
|
}
|
|
}
|
|
|
|
/** Relative path of non-empty segments, without ".", "..", NUL, backslash or control characters. */
|
|
public static function isSafePath(string $path): bool {
|
|
if ($path === '' || str_starts_with($path, '/') || preg_match('/[\x00-\x1f\\\\]/', $path)) {
|
|
return false;
|
|
}
|
|
foreach (explode('/', $path) as $segment) {
|
|
if ($segment === '' || $segment === '.' || $segment === '..') {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/** Small encrypted objects (manifests). */
|
|
public function putObject(string $path, string $data): void {
|
|
self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path));
|
|
}
|
|
|
|
public function getObject(string $path): string {
|
|
$fh = $this->backend->get($path);
|
|
$data = $this->cipher->decryptString((string)stream_get_contents($fh), 'object:' . $path);
|
|
fclose($fh);
|
|
return $data;
|
|
}
|
|
|
|
/**
|
|
* Move a snapshot (and its database dump) to the trash on the location. Its data stays in
|
|
* use until purgeTrash() removes it after the deletion delay, so a malicious or mistaken
|
|
* cleanup can be undone with untrash() within that time.
|
|
*/
|
|
public function forget(string $snapshotId, string $by = ''): void {
|
|
$meta = $this->snapshotMeta($snapshotId);
|
|
$dbInTrash = null;
|
|
if (isset($meta['db']) && is_string($meta['db']) && $this->backend->exists($meta['db'])) {
|
|
// Its encryption is bound to its path (object:db/...), which changes on a move, so it
|
|
// must be decrypted and re-encrypted under the new path, not moved as raw bytes.
|
|
$dbInTrash = 'trash/' . $meta['db'];
|
|
$this->putObject($dbInTrash, $this->getObject($meta['db']));
|
|
}
|
|
// Copy first, switch the catalog, delete the originals last: a crash at any point leaves
|
|
// either the old state or the new one plus unreferenced leftovers, never a catalog entry
|
|
// pointing at a missing object. The snapshot object is bound to its id (not its path),
|
|
// so it can be copied as raw bytes.
|
|
$this->copyRaw('snapshots/' . $snapshotId, 'trash/snapshots/' . $snapshotId);
|
|
$this->putObject('trash/info/' . $snapshotId, json_encode(['forgottenAt' => time(), 'by' => $by, 'db' => $meta['db'] ?? null,
|
|
'dbInTrash' => $dbInTrash, 'time' => $meta['time'] ?? null, 'label' => $meta['label'] ?? ''], JSON_THROW_ON_ERROR));
|
|
unset($this->catalog->snapshots[$snapshotId]);
|
|
$this->catalog->trash[$snapshotId] = true;
|
|
$this->catalog->write();
|
|
if ($dbInTrash !== null) {
|
|
$this->backend->delete($meta['db']);
|
|
}
|
|
$this->backend->delete('snapshots/' . $snapshotId);
|
|
}
|
|
|
|
/** Undo forget(): move a snapshot back from the trash. */
|
|
public function untrash(string $snapshotId): void {
|
|
$info = $this->trashInfo($snapshotId);
|
|
$restoreDb = $info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash']);
|
|
if ($restoreDb) {
|
|
$this->putObject($info['db'], $this->getObject($info['dbInTrash']));
|
|
}
|
|
// Same order as forget(): copy, switch the catalog, then delete the trash copies.
|
|
$this->copyRaw('trash/snapshots/' . $snapshotId, 'snapshots/' . $snapshotId);
|
|
$this->catalog->snapshots[$snapshotId] = true;
|
|
unset($this->catalog->trash[$snapshotId]);
|
|
$this->catalog->write();
|
|
if ($restoreDb) {
|
|
$this->backend->delete($info['dbInTrash']);
|
|
}
|
|
$this->backend->delete('trash/snapshots/' . $snapshotId);
|
|
$this->backend->delete('trash/info/' . $snapshotId);
|
|
}
|
|
|
|
/** Copy an object as raw bytes, buffered locally (remote read streams cannot be uploaded directly everywhere). */
|
|
private function copyRaw(string $from, string $to): void {
|
|
$in = $this->backend->get($from);
|
|
$buffer = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
if ($buffer === false) {
|
|
fclose($in);
|
|
throw new RepositoryException("Cannot buffer $from");
|
|
}
|
|
try {
|
|
if (stream_copy_to_stream($in, $buffer) === false) {
|
|
throw new RepositoryException("Cannot read $from");
|
|
}
|
|
} finally {
|
|
fclose($in);
|
|
}
|
|
rewind($buffer);
|
|
$this->backend->put($to, $buffer);
|
|
}
|
|
|
|
/** Record a user_migration export's manifest in the catalog (rollback-protected like a snapshot). */
|
|
public function recordUserExport(string $manifestPath): void {
|
|
$this->catalog->userExports[$manifestPath] = true;
|
|
$this->catalog->write();
|
|
}
|
|
|
|
/**
|
|
* User export manifests according to the verified catalog, newest first. A manifest the
|
|
* catalog lists but the location no longer has means it was deleted or hidden behind NG
|
|
* Backup's back, same as a missing snapshot (see listSnapshots()).
|
|
*
|
|
* @return list<string>
|
|
*/
|
|
public function listUserExports(?string $uid = null): array {
|
|
$prefix = $uid !== null ? 'users/' . $uid . '/' : 'users/';
|
|
$out = [];
|
|
foreach (array_keys($this->catalog->userExports) as $path) {
|
|
if (!str_starts_with($path, $prefix)) {
|
|
continue;
|
|
}
|
|
if (!$this->backend->exists($path)) {
|
|
throw new RepositoryException("User export $path is missing on the location (deleted or hidden outside NG Backup)");
|
|
}
|
|
$out[] = $path;
|
|
}
|
|
sort($out, SORT_STRING);
|
|
return array_reverse($out);
|
|
}
|
|
|
|
/** Whether $manifestPath is a user export the verified catalog actually recorded. */
|
|
public function isRecordedUserExport(string $manifestPath): bool {
|
|
return isset($this->catalog->userExports[$manifestPath]);
|
|
}
|
|
|
|
/**
|
|
* The recorded export $ref names: a manifest path (users/<uid>/<exportId>) as is, or a bare
|
|
* user id (also "users/<uid>") when that user has exactly one export here. Anything else
|
|
* throws, with the exports to choose from in the message.
|
|
*/
|
|
public function resolveUserExport(string $ref): string {
|
|
if ($this->isRecordedUserExport($ref)) {
|
|
return $ref;
|
|
}
|
|
$rest = trim($ref, '/');
|
|
if (str_starts_with($rest, 'users/')) {
|
|
$rest = substr($rest, strlen('users/'));
|
|
}
|
|
$uid = explode('/', $rest)[0];
|
|
$exports = $uid !== '' ? $this->listUserExports($uid) : [];
|
|
if (!str_contains($rest, '/') && count($exports) === 1) {
|
|
return $exports[0];
|
|
}
|
|
if ($exports === []) {
|
|
throw new RepositoryException("$ref is not a recorded user export for this location, and $uid has no exports here. Expected users/<uid>/<exportId>; occ backup:user:list shows the exports");
|
|
}
|
|
$which = str_contains($rest, '/') ? "$ref is not a recorded user export for this location" : "$uid has " . count($exports) . ' exports here';
|
|
throw new RepositoryException("$which; pass one of these (newest first):\n " . implode("\n ", $exports));
|
|
}
|
|
|
|
/** Permanently delete a user export. Its blobs are freed by the next prune() like any other. */
|
|
public function forgetUserExport(string $manifestPath): void {
|
|
if (!isset($this->catalog->userExports[$manifestPath])) {
|
|
throw new RepositoryException("$manifestPath is not a recorded user export");
|
|
}
|
|
unset($this->catalog->userExports[$manifestPath]);
|
|
$this->catalog->write();
|
|
$this->backend->delete($manifestPath);
|
|
}
|
|
|
|
/** @return array<string, array{forgottenAt:int, by:string, db:?string, dbInTrash:?string, time:?string, label:string}> */
|
|
public function trash(): array {
|
|
$out = [];
|
|
foreach (array_keys($this->catalog->trash) as $id) {
|
|
try {
|
|
$out[$id] = $this->trashInfo($id);
|
|
} catch (\Throwable) {
|
|
throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)");
|
|
}
|
|
}
|
|
return $out;
|
|
}
|
|
|
|
/** Permanently delete trash entries older than $delaySeconds. @return list<string> purged snapshot ids */
|
|
public function purgeTrash(int $delaySeconds): array {
|
|
$doomed = [];
|
|
foreach ($this->trash() as $id => $info) {
|
|
if (time() - $info['forgottenAt'] < $delaySeconds) {
|
|
continue;
|
|
}
|
|
$doomed[$id] = $info;
|
|
unset($this->catalog->trash[$id]);
|
|
}
|
|
if ($doomed === []) {
|
|
return [];
|
|
}
|
|
$this->catalog->write();
|
|
foreach ($doomed as $id => $info) {
|
|
if ($info['dbInTrash'] !== null) {
|
|
$this->backend->delete($info['dbInTrash']);
|
|
}
|
|
$this->backend->delete('trash/snapshots/' . $id);
|
|
$this->backend->delete('trash/info/' . $id);
|
|
}
|
|
return array_keys($doomed);
|
|
}
|
|
|
|
private function trashInfo(string $snapshotId): array {
|
|
return json_decode($this->getObject('trash/info/' . $snapshotId), true, 512, JSON_THROW_ON_ERROR);
|
|
}
|
|
|
|
/**
|
|
* Delete packs no snapshot or user export uses any more, and repack packs where more than
|
|
* $repackThreshold of the bytes are unused. Must not run while a backup to this repository
|
|
* is in progress (its blobs are not referenced by a snapshot yet); callers hold the lock.
|
|
*
|
|
* @return array{packs:int, deleted:int, repacked:int, freedBytes:int, keptBlobs:int, unusedBlobs:int}
|
|
*/
|
|
public function prune(bool $dryRun = false, float $repackThreshold = 0.5, array $ignoreSnapshots = []): array {
|
|
// Deleting is decided on the authenticated index files on the location, never on the local cache.
|
|
$this->index->load(true);
|
|
$used = [];
|
|
$ignore = array_flip($ignoreSnapshots); // e.g. snapshots a dry run would forget first
|
|
foreach ($this->listSnapshots() as $sid) {
|
|
if (isset($ignore[$sid])) {
|
|
continue;
|
|
}
|
|
foreach ($this->streamEntries($sid) as $e) {
|
|
foreach ($e['b'] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
$meta = $this->snapshotMeta($sid);
|
|
if (isset($meta['db']) && $this->backend->exists($meta['db'])) {
|
|
foreach (json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
|
foreach ($t['blobs'] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Snapshots in the trash keep their data until purgeTrash() removed them.
|
|
foreach ($this->trash() as $sid => $info) {
|
|
foreach ($this->streamEntries($sid, 'trash/snapshots/') as $e) {
|
|
foreach ($e['b'] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
if ($info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash'])) {
|
|
foreach (json_decode($this->getObject($info['dbInTrash']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
|
foreach ($t['blobs'] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// listUserExports() throws when a recorded export's manifest is missing: then stop, since
|
|
// freeing its blobs would turn a tampering or storage alarm into irreversible deletion.
|
|
$exports = array_unique(array_merge($this->listUserExports(), $this->backend->list('users')));
|
|
foreach ($exports as $path) {
|
|
foreach (json_decode($this->getObject($path), true, 512, JSON_THROW_ON_ERROR)['entries'] as $e) {
|
|
foreach ($e['b'] ?? [] as $b) {
|
|
$used[$b] = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
$stats = ['packs' => 0, 'deleted' => 0, 'repacked' => 0, 'freedBytes' => 0, 'keptBlobs' => count($used), 'unusedBlobs' => 0];
|
|
$packs = $this->index->packs();
|
|
$stats['packs'] = count($packs);
|
|
$writer = null;
|
|
$toRemove = [];
|
|
foreach ($packs as $packId => $blobs) {
|
|
$total = 0;
|
|
$unused = 0;
|
|
foreach ($blobs as $b) {
|
|
$total += $b['length'];
|
|
if (!isset($used[$b['id']])) {
|
|
$unused += $b['length'];
|
|
$stats['unusedBlobs']++;
|
|
}
|
|
}
|
|
if ($unused === 0 || ($unused < $total && $unused / $total <= $repackThreshold)) {
|
|
continue;
|
|
}
|
|
$stats['freedBytes'] += $unused;
|
|
if ($unused === $total) {
|
|
$stats['deleted']++;
|
|
} else {
|
|
$stats['repacked']++;
|
|
if (!$dryRun) {
|
|
// No size limit here: repacking is how space is freed.
|
|
$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
|
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
|
foreach ($blobs as $b) {
|
|
if (isset($used[$b['id']])) {
|
|
[, , , $raw, $flags] = $this->index->get($b['id']);
|
|
$writer->addRaw($b['id'], $this->backend->getRange($path, $b['offset'], $b['length']), $raw, $flags);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
$toRemove[] = $packId;
|
|
}
|
|
if ($dryRun) {
|
|
return $stats;
|
|
}
|
|
// New packs and their index entries are durable before any old pack is removed.
|
|
$writer?->flush();
|
|
foreach ($toRemove as $packId) {
|
|
$this->index->removePack($packId);
|
|
$this->backend->delete('packs/' . substr($packId, 0, 2) . '/' . $packId);
|
|
}
|
|
return $stats;
|
|
}
|
|
|
|
/**
|
|
* Snapshots according to the verified catalog. A snapshot the catalog lists but the location
|
|
* no longer has means it was deleted or hidden behind NG Backup's back.
|
|
*
|
|
* @return list<string>
|
|
*/
|
|
public function listSnapshots(): array {
|
|
$present = array_flip(array_map('basename', $this->backend->listFiles('snapshots')));
|
|
$listed = array_keys($this->catalog->snapshots);
|
|
foreach ($listed as $id) {
|
|
if (!isset($present[$id])) {
|
|
throw new RepositoryException("Snapshot $id is missing on the location (deleted or hidden outside NG Backup)");
|
|
}
|
|
}
|
|
sort($listed);
|
|
return $listed;
|
|
}
|
|
|
|
public function blobCount(): int {
|
|
return $this->index->count();
|
|
}
|
|
|
|
/**
|
|
* Read a file into blobs and make sure it did not change while reading (size, mtime, ctime
|
|
* before and after). A file that keeps changing is an error rather than a torn backup.
|
|
*
|
|
* @return list<string>
|
|
*/
|
|
private function readStable(string $path, PackWriter $packs, array &$stats): array {
|
|
for ($attempt = 1; $attempt <= 3; $attempt++) {
|
|
$before = self::fileVersion($path);
|
|
$blobs = [];
|
|
$fh = fopen($path, 'rb');
|
|
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
|
|
$blobs[] = $this->storeBlob($data, $packs, $stats);
|
|
}
|
|
fclose($fh);
|
|
if (self::fileVersion($path) === $before) {
|
|
return $blobs;
|
|
}
|
|
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
|
}
|
|
throw new RepositoryException('File keeps changing while being read: ' . $path);
|
|
}
|
|
|
|
/** Cheap file identity: changes when the content may have changed. */
|
|
public static function fileVersion(string $path): string {
|
|
clearstatcache(true, $path);
|
|
$st = @stat($path);
|
|
return $st === false ? 'missing' : implode(':', [$st['size'], $st['mtime'], $st['ctime'], $st['ino']]);
|
|
}
|
|
|
|
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
|
|
$id = $this->keys->blobId($data);
|
|
if ($this->index->has($id)) {
|
|
$stats['dupBlobs'] = ($stats['dupBlobs'] ?? 0) + 1;
|
|
return $id;
|
|
}
|
|
$flags = 0;
|
|
$payload = $data;
|
|
$deflated = self::worthDeflating($data) ? gzdeflate($data, 6) : false;
|
|
if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) {
|
|
$payload = $deflated;
|
|
$flags |= self::FLAG_DEFLATE;
|
|
}
|
|
$stats['uploaded'] = ($stats['uploaded'] ?? 0) + $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
|
|
$stats['newBlobs'] = ($stats['newBlobs'] ?? 0) + 1;
|
|
return $id;
|
|
}
|
|
|
|
/**
|
|
* Photos, video and archives are compressed already: deflating them takes most of a backup's
|
|
* CPU time (about 24 MiB/s) only to be thrown away, as it saves less than the 5% that keeps
|
|
* it. A fast deflate of three small slices tells first; below 10% savings there, the full
|
|
* deflate is skipped.
|
|
*/
|
|
private static function worthDeflating(string $data): bool {
|
|
$n = strlen($data);
|
|
if ($n < self::DEFLATE_PROBE_FROM) {
|
|
return true;
|
|
}
|
|
$k = self::DEFLATE_PROBE_SLICE;
|
|
$probe = substr($data, 0, $k) . substr($data, intdiv($n - $k, 2), $k) . substr($data, $n - $k);
|
|
$deflated = gzdeflate($probe, 1);
|
|
return $deflated !== false && strlen($deflated) * 10 < strlen($probe) * 9;
|
|
}
|
|
|
|
/**
|
|
* Blobs spread over many packs (the tree segments of a run, one per step), through
|
|
* IBackend::getRanges(): on a remote location one at a time is a round trip each, which for
|
|
* a long run made writing its snapshot take a quarter of an hour. Blobs that follow each
|
|
* other in a pack are read as one range.
|
|
*
|
|
* @param list<string> $ids
|
|
* @return \Generator<int, string> verified blob contents, in order
|
|
*/
|
|
private function loadBlobs(array $ids): \Generator {
|
|
/** @var list<array{0:string, 1:int, 2:int}> $ranges [path, offset, length] of each read */
|
|
$ranges = [];
|
|
/** @var list<list<array{0:string, 1:int, 2:int}>> $parts per read: [blob id, offset in the read, length] */
|
|
$parts = [];
|
|
// the read being extended
|
|
$curPath = null;
|
|
$curOffset = 0;
|
|
$curLength = 0;
|
|
$curParts = [];
|
|
foreach ($ids as $id) {
|
|
[$pack, $offset, $length] = $this->index->get($id);
|
|
$path = 'packs/' . substr($pack, 0, 2) . '/' . $pack;
|
|
if ($curPath === $path && $curOffset + $curLength === $offset && $curLength + $length <= self::MERGED_READ_MAX) {
|
|
$curParts[] = [$id, $curLength, $length];
|
|
$curLength += $length;
|
|
continue;
|
|
}
|
|
if ($curPath !== null) {
|
|
$ranges[] = [$curPath, $curOffset, $curLength];
|
|
$parts[] = $curParts;
|
|
}
|
|
[$curPath, $curOffset, $curLength, $curParts] = [$path, $offset, $length, [[$id, 0, $length]]];
|
|
}
|
|
if ($curPath !== null) {
|
|
$ranges[] = [$curPath, $curOffset, $curLength];
|
|
$parts[] = $curParts;
|
|
}
|
|
$done = 0;
|
|
try {
|
|
foreach ($this->backend->getRanges($ranges) as $i => $data) {
|
|
foreach ($parts[$i] as [$id, $at, $length]) {
|
|
try {
|
|
$blob = $this->decodeBlob($id, substr($data, $at, $length));
|
|
} catch (\Exception) {
|
|
// a read that went wrong on the way (see loadBlob()): this blob on its own
|
|
$blob = $this->loadBlob($id);
|
|
// its pack stream is closed before the next range is read on the same connection
|
|
$this->packReader->close();
|
|
}
|
|
yield $blob;
|
|
$done++;
|
|
}
|
|
}
|
|
} catch (BackendException) {
|
|
// A pack moved, or the read failed for a while (see loadBlob()): the rest one blob at a
|
|
// time, each read once more after a failure.
|
|
foreach (array_slice($ids, $done) as $id) {
|
|
yield $this->loadBlob($id);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** A blob for restore(): read ahead when it could be, otherwise (or when that copy is bad) the usual way. */
|
|
private function restoreBlob(string $id, ?ReadAheadPlan $plan): string {
|
|
if ($plan !== null) {
|
|
[$pack, $offset, $length] = $this->index->get($id);
|
|
$encrypted = $plan->read('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
|
if ($encrypted !== null) {
|
|
try {
|
|
return $this->decodeBlob($id, $encrypted);
|
|
} catch (\Exception) {
|
|
// read again from the location below, which reports what is wrong
|
|
}
|
|
}
|
|
}
|
|
return $this->loadBlob($id);
|
|
}
|
|
|
|
private function loadBlob(string $id): string {
|
|
[$pack, $offset, $length] = $this->index->get($id);
|
|
try {
|
|
$encrypted = $this->packReader->read($pack, $offset, $length);
|
|
} catch (BackendException $e) {
|
|
// Another server's cleanup may have repacked this blob into a new pack since the index
|
|
// was loaded (a disaster recovery test while the original server still runs): add the
|
|
// new index files, and read the blob where it is now. Or the read failed for a while
|
|
// (a connection that died; the backend has replaced it by now): then read it once
|
|
// more where it was. A second failure is the real one.
|
|
try {
|
|
$this->index->refresh();
|
|
} catch (\Throwable) {
|
|
// the location could not be listed either: read again where the blob was
|
|
}
|
|
[$moved, $offset, $length] = $this->index->get($id);
|
|
try {
|
|
$encrypted = $this->packReader->read($moved, $offset, $length);
|
|
} catch (BackendException) {
|
|
throw $e;
|
|
}
|
|
$pack = $moved;
|
|
}
|
|
try {
|
|
return $this->decodeBlob($id, $encrypted);
|
|
} catch (\Exception $damaged) {
|
|
// Bytes that do not decrypt: damaged on the location, or a read that went wrong on the
|
|
// way (a stream that lost its place: an SMB restore stopped after 17 hours like this).
|
|
// Read exactly this blob once more, on a request of its own; only a second failure
|
|
// means the data itself is damaged.
|
|
$this->packReader->close();
|
|
try {
|
|
$again = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
|
} catch (\Exception) {
|
|
throw $damaged;
|
|
}
|
|
$data = $this->decodeBlob($id, $again);
|
|
if (function_exists('OCP\Log\logger')) {
|
|
\OCP\Log\logger('ng_backup')->warning('NG Backup: blob ' . $id . ' did not decrypt as first read from pack ' . $pack . ' at ' . $offset . '; read again, it is fine. First read: ' . self::howItDiffered($encrypted, $again));
|
|
}
|
|
return $data;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* What a read that went wrong looked like next to the right bytes, for the log: where it starts
|
|
* to differ, and whether the rest is zeros or the right bytes from another place (a stream that
|
|
* lost its place).
|
|
*/
|
|
private static function howItDiffered(string $wrong, string $right): string {
|
|
$at = strspn($wrong ^ $right, "\0");
|
|
$text = strlen($wrong) . ' bytes, differ from byte ' . $at . ' of ' . strlen($right);
|
|
$tail = substr($wrong, $at, 64);
|
|
if ($tail === '') {
|
|
return $text . ' (too short)';
|
|
}
|
|
if (trim($tail, "\0") === '') {
|
|
return $text . ' (zeros)';
|
|
}
|
|
$found = strpos($right, $tail);
|
|
return $text . ($found !== false ? ' (the right bytes from ' . ($found - $at) . ' bytes further on)' : ' (other bytes)');
|
|
}
|
|
|
|
/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */
|
|
private function decodeBlob(string $id, string $encrypted): string {
|
|
[, , , $raw, $flags] = $this->index->get($id);
|
|
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
|
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
|
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
|
throw new RepositoryException('Blob verification failed: ' . $id);
|
|
}
|
|
return $data;
|
|
}
|
|
|
|
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
|
private function loadEntries(string $snapshotId): array {
|
|
$map = [];
|
|
foreach ($this->streamEntries($snapshotId) as $e) {
|
|
$map[$e['p']] = ['s' => $e['s'], 'm' => $e['m'], 'b' => $e['b']];
|
|
}
|
|
return $map;
|
|
}
|
|
|
|
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> entries of a snapshot, in stored order */
|
|
public function entries(string $snapshotId): \Generator {
|
|
return $this->streamEntries($snapshotId);
|
|
}
|
|
|
|
/**
|
|
* Checksum audit of a snapshot: every blob it references (files and the database dump, when
|
|
* the snapshot has one) must be in the index, freshly re-downloaded and re-authenticated from
|
|
* the location rather than trusted from the local cache (same reasoning as prune()), and its
|
|
* pack must still be present. With $deep, each blob is also downloaded and decrypted, which
|
|
* verifies its AEAD authentication tag and its content hash against the blob id (loadBlob()
|
|
* already does both on every real read -- this just does it for everything in the snapshot,
|
|
* deliberately, rather than relying on it to come up during a later restore). A blob
|
|
* referenced by more than one file is only checked once; a missing database manifest is a
|
|
* failure, not something to silently skip over.
|
|
*
|
|
* @param ?callable(array{phase:string, done:int, total:?int, blobs?:int, bytes?:int}):void $heartbeat
|
|
* called often during the check, with how far it is: phase "index" (index files
|
|
* downloaded), "packs" (pack folders listed), "files" and "database" (files of the
|
|
* snapshot checked); callers use it to keep a time-boxed lease alive and to report
|
|
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
|
*/
|
|
public function verify(string $snapshotId, bool $deep = false, ?callable $heartbeat = null): array {
|
|
$heartbeat ??= static function (array $progress): void {
|
|
};
|
|
$meta = $this->snapshotMeta($snapshotId);
|
|
$filesTotal = isset($meta['stats']['files']) ? (int)$meta['stats']['files'] : null;
|
|
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
|
|
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
|
|
});
|
|
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
|
$seen = [];
|
|
[$packSeen, $listedDirs] = $this->presentPacks($heartbeat);
|
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
|
$stats['filesChecked']++;
|
|
foreach ($entry['b'] as $id) {
|
|
if (isset($seen[$id])) {
|
|
continue;
|
|
}
|
|
$seen[$id] = true;
|
|
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
|
|
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
|
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
}
|
|
}
|
|
if (isset($meta['db']) && is_string($meta['db'])) {
|
|
if (!$this->backend->exists($meta['db'])) {
|
|
$stats['missing'][] = $meta['db'];
|
|
} else {
|
|
$dbManifest = json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
|
foreach ($dbManifest['tables'] as $info) {
|
|
foreach ($info['blobs'] as $id) {
|
|
if (isset($seen[$id])) {
|
|
continue;
|
|
}
|
|
$seen[$id] = true;
|
|
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
|
|
$heartbeat(['phase' => 'database', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
|
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return $stats;
|
|
}
|
|
|
|
/**
|
|
* Which packs are on the location, from one listing per pack folder instead of a request per
|
|
* pack. A folder that cannot be listed, or comes back empty (which a storage also reports when
|
|
* it could not look), is left out: its packs are then checked one by one. With no more packs
|
|
* than folders nothing is listed: a request per pack is fewer requests then, and on FTP every
|
|
* request takes a data connection, of which a server may have only a few.
|
|
*
|
|
* @return array{0: array<string, bool>, 1: array<string, true>} pack id => present, and the
|
|
* folders that were listed
|
|
*/
|
|
private function presentPacks(callable $heartbeat): array {
|
|
$present = [];
|
|
$listed = [];
|
|
if ($this->index->packCount() <= $this->listPacksAbove) {
|
|
return [$present, $listed];
|
|
}
|
|
for ($i = 0; $i < 256; $i++) {
|
|
$dir = sprintf('%02x', $i);
|
|
try {
|
|
$files = $this->backend->listFiles('packs/' . $dir);
|
|
} catch (BackendException) {
|
|
$files = [];
|
|
}
|
|
foreach ($files as $path) {
|
|
$present[basename($path)] = true;
|
|
}
|
|
if ($files !== []) {
|
|
$listed[$dir] = true;
|
|
}
|
|
$heartbeat(['phase' => 'packs', 'done' => $i + 1, 'total' => 256]);
|
|
}
|
|
return [$present, $listed];
|
|
}
|
|
|
|
/**
|
|
* Same checks as verify(), for a user export: every blob its manifest references must be in
|
|
* the freshly re-authenticated index with its pack present; $deep also decrypts each.
|
|
*
|
|
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
|
*/
|
|
public function verifyUserExport(string $manifestPath, bool $deep = false, ?callable $heartbeat = null): array {
|
|
if (!$this->isRecordedUserExport($manifestPath)) {
|
|
throw new RepositoryException("$manifestPath is not a recorded user export for this location");
|
|
}
|
|
$heartbeat ??= static function (array $progress): void {
|
|
};
|
|
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
|
|
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
|
|
});
|
|
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
|
$seen = [];
|
|
$packSeen = [];
|
|
$manifest = json_decode($this->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
|
foreach ($manifest['entries'] as $entry) {
|
|
if (($entry['t'] ?? '') !== 'f') {
|
|
continue;
|
|
}
|
|
$stats['filesChecked']++;
|
|
foreach ($entry['b'] ?? [] as $id) {
|
|
if (isset($seen[$id])) {
|
|
continue;
|
|
}
|
|
$seen[$id] = true;
|
|
$this->verifyBlob($id, $deep, $stats, $packSeen);
|
|
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => null,
|
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
}
|
|
}
|
|
return $stats;
|
|
}
|
|
|
|
/**
|
|
* @param array<string, bool> $packSeen pack id => present; filled in as packs are checked
|
|
* @param array<string, true> $listedDirs pack folders whose listing is in $packSeen: a pack
|
|
* not in it is not there
|
|
*/
|
|
private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen, array $listedDirs = []): void {
|
|
if (!$this->index->has($id)) {
|
|
$stats['missing'][] = $id;
|
|
return;
|
|
}
|
|
[$pack] = $this->index->get($id);
|
|
$packExists = $packSeen[$pack] ??= isset($listedDirs[substr($pack, 0, 2)])
|
|
? false
|
|
: $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
|
|
if (!$packExists) {
|
|
$stats['missing'][] = $id;
|
|
return;
|
|
}
|
|
$stats['blobsChecked']++;
|
|
if (!$deep) {
|
|
return;
|
|
}
|
|
// Reading and checking are kept apart: a read that fails is usually a network or
|
|
// storage error, so it is retried and reported as unreadable (a pack that stays
|
|
// unreadable may still be truncated). Only a blob that was read but does not
|
|
// decrypt or match its id counts as failed: that is proof of damage.
|
|
[, $offset, $length] = $this->index->get($id);
|
|
$encrypted = null;
|
|
for ($attempt = 1; $attempt <= self::VERIFY_READ_ATTEMPTS && $encrypted === null; $attempt++) {
|
|
try {
|
|
$encrypted = $this->packReader->read($pack, $offset, $length);
|
|
} catch (\Exception $e) { // storage adapters throw their own exceptions; \Error (a bug) is not retried
|
|
$stats['readError'] = $e->getMessage();
|
|
}
|
|
}
|
|
if ($encrypted === null) {
|
|
$stats['unreadable'][] = $id;
|
|
return;
|
|
}
|
|
try {
|
|
$stats['bytesChecked'] += strlen($this->decodeBlob($id, $encrypted));
|
|
} catch (\Exception) { // damage; an \Error is a bug and must not be reported as damage
|
|
$stats['failed'][] = $id;
|
|
}
|
|
}
|
|
|
|
/** The meta record (first line) of a snapshot. */
|
|
public function snapshotMeta(string $snapshotId): array {
|
|
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$fh = $this->backend->get('snapshots/' . $snapshotId);
|
|
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
|
fclose($fh);
|
|
rewind($plain);
|
|
$meta = json_decode((string)fgets($plain), true, 512, JSON_THROW_ON_ERROR);
|
|
fclose($plain);
|
|
return $meta;
|
|
}
|
|
|
|
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
|
private function streamEntries(string $snapshotId, string $dir = 'snapshots/'): \Generator {
|
|
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
$fh = $this->backend->get($dir . $snapshotId);
|
|
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
|
fclose($fh);
|
|
rewind($plain);
|
|
fgets($plain); // meta line
|
|
while (($line = fgets($plain)) !== false) {
|
|
$entry = json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
|
if (!is_array($entry) || !is_string($entry['p'] ?? null) || !self::isSafePath($entry['p'])
|
|
|| !is_int($entry['s'] ?? null) || !is_array($entry['b'] ?? null)) {
|
|
throw new RepositoryException('Invalid entry in snapshot ' . $snapshotId);
|
|
}
|
|
yield $entry;
|
|
}
|
|
fclose($plain);
|
|
}
|
|
|
|
/** @return \Generator<string, \SplFileInfo> relative path => file, in a stable order */
|
|
private function walk(string $root): \Generator {
|
|
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
|
|
$files = [];
|
|
foreach ($it as $file) {
|
|
if ($file->isFile()) {
|
|
$files[substr($file->getPathname(), strlen($root) + 1)] = $file;
|
|
}
|
|
}
|
|
ksort($files, SORT_STRING);
|
|
yield from $files;
|
|
}
|
|
|
|
private static function putString(IBackend $backend, string $path, string $data): void {
|
|
$s = fopen('php://memory', 'w+b');
|
|
fwrite($s, $data);
|
|
rewind($s);
|
|
$backend->put($path, $s);
|
|
}
|
|
|
|
private static function readFull($fh, int $length): string {
|
|
$buffer = '';
|
|
while (strlen($buffer) < $length && !feof($fh)) {
|
|
$buffer .= (string)fread($fh, $length - strlen($buffer));
|
|
}
|
|
return $buffer;
|
|
}
|
|
}
|