<?php

declare(strict_types=1);

// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later

namespace OCA\NgBackup\Repository;

use OCA\NgBackup\Backend\BackendException;
use OCA\NgBackup\Backend\IBackend;
use OCA\NgBackup\Backend\IReadsAhead;
use OCA\NgBackup\Crypto\KeyRing;
use OCA\NgBackup\Crypto\StreamCipher;

/**
 * Content-addressed, encrypted, deduplicating repository.
 *
 * Layout on the backend:
 *   config                 plaintext JSON: format version, repository id, wrapped master key
 *   packs/xx/<packId>      encrypted blobs + encrypted pack header
 *   index/<packId>         encrypted list of the blobs in that pack
 *   snapshots/<snapId>     encrypted JSON lines: one meta line, then one line per file
 *
 * Files are read as streams and cut into blobs of BLOB_SIZE; only one blob and one pack are in
 * memory at a time. A blob that already exists (same keyed hash) is not stored again.
 */
final class Repository {
	public const FORMAT = 1;
	public const BLOB_SIZE = 4 * 1048576;
	private const FLAG_DEFLATE = 1;
	/** Blobs from this size on get a quick compression test before the full compression. */
	private const DEFLATE_PROBE_FROM = 256 * 1024;
	/** Bytes of each of the three slices (start, middle, end) in that test. */
	private const DEFLATE_PROBE_SLICE = 32 * 1024;
	/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
	private const VERIFY_READ_ATTEMPTS = 3;
	/** Largest read that loadBlobs() makes of blobs that follow each other in a pack. */
	private const MERGED_READ_MAX = 16 * 1048576;

	private StreamCipher $cipher;
	private BlobIndex $index;
	private ?PackWriter $runPacks = null;
	private ?int $maxBytes = null;
	private Catalog $catalog;
	private PackReader $packReader;
	/** @var array<string, array<string, mixed>> meta record of the snapshot read last */
	private array $lastMeta = [];
	/** verify() lists the pack folders from this many packs on (see presentPacks()); tests lower it. */
	public int $listPacksAbove = 256;

	private function __construct(
		private IBackend $backend,
		private KeyRing $keys,
		private string $repositoryId,
	) {
		$this->cipher = new StreamCipher($keys);
		$this->packReader = new PackReader($backend);
		$this->backend = new PackReaderGuard($backend, $this->packReader);
		$this->index = new BlobIndex($this->backend, $this->cipher);
	}

	public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
		$keys = KeyRing::generate();
		return self::initWithKey($backend, $keys, $keys->wrap($passphrase));
	}

	/**
	 * Create a repository with an existing master key. $wrappedKey is the passphrase-wrapped
	 * copy (KeyRing::wrap) stored in the repository config, so the repository can be opened
	 * with the passphrase from the recovery kit even when the server is gone.
	 */
	public static function initWithKey(IBackend $backend, KeyRing $keys, array $wrappedKey, ?ICatalogAnchor $anchor = null): self {
		if ($backend->exists('config')) {
			throw new RepositoryException('A repository already exists at this location');
		}
		// No config, but backup data: the config is missing (or could not be seen just now, a
		// connection that failed). A new repository here would write a config over the one that
		// opens this data; refuse instead.
		foreach (['index', 'snapshots', 'trash/snapshots', 'catalog'] as $dir) {
			if ($backend->listFiles($dir) !== []) {
				throw new RepositoryException("This location holds backup data ($dir/) but no repository config could be read: not creating a new repository over it. Check the connection and the folder; a missing config can be restored from the recovery kit.");
			}
		}
		$id = bin2hex(random_bytes(16));
		$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
			'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))];
		self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
		$repo = new self($backend, $keys, $id);
		$repo->loadCatalog($anchor);
		return $repo;
	}

	/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
	/** @param ?callable(int, int): void $progress index files downloaded so far, and in all (see BlobIndex::load()) */
	public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null, ?callable $progress = null): self {
		$config = self::readConfig($backend);
		try {
			$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
		} catch (\OCA\NgBackup\Crypto\CryptoException) {
			$ok = false;
		}
		if (!$ok) {
			throw new RepositoryException('This repository was created with a different key');
		}
		$repo = new self($backend, $keys, $config['id']);
		if ($cache !== null) {
			$repo->index->setCache($cache, $config['id']);
		}
		$repo->index->load(false, $progress);
		$repo->loadCatalog($anchor);
		return $repo;
	}

	private function loadCatalog(?ICatalogAnchor $anchor): void {
		$this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor);
		$this->catalog->load(
			fn () => array_map('basename', $this->backend->listFiles('snapshots')),
			fn () => array_map('basename', $this->backend->listFiles('trash/info')),
			fn () => $this->backend->list('users'));
	}

	/** Refuse uploads that would make the stored data larger than this (null = no limit). */
	public function setMaxBytes(?int $maxBytes): void {
		$this->maxBytes = $maxBytes;
	}

	public function storedBytes(): int {
		return $this->index->storedBytes();
	}

	/** Current catalog generation (for status and tests). */
	public function catalogGeneration(): int {
		return $this->catalog->gen;
	}

	/** @return list<string> snapshots the retention policy leaves alone */
	public function pinnedSnapshots(): array {
		return array_map('strval', array_keys($this->catalog->pinned));
	}

	/** @return array<string, string> snapshot id => note */
	public function snapshotNotes(): array {
		return $this->catalog->notes;
	}

	public function setPinned(string $snapshotId, bool $pinned): void {
		$this->requireSnapshot($snapshotId);
		if (isset($this->catalog->pinned[$snapshotId]) === $pinned) {
			return;
		}
		if ($pinned) {
			$this->catalog->pinned[$snapshotId] = true;
		} else {
			unset($this->catalog->pinned[$snapshotId]);
		}
		$this->catalog->write();
	}

	/** A note kept in the catalog: unlike the label inside the snapshot, it can change afterwards. */
	public function setSnapshotNote(string $snapshotId, string $note): void {
		$this->requireSnapshot($snapshotId);
		if (($this->catalog->notes[$snapshotId] ?? '') === $note) {
			return;
		}
		if ($note === '') {
			unset($this->catalog->notes[$snapshotId]);
		} else {
			$this->catalog->notes[$snapshotId] = $note;
		}
		$this->catalog->write();
	}

	private function requireSnapshot(string $snapshotId): void {
		if (!isset($this->catalog->snapshots[$snapshotId])) {
			throw new \InvalidArgumentException('There is no snapshot ' . $snapshotId . ' on this location');
		}
	}

	/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
	public function indexDownloads(): int {
		return $this->index->downloaded;
	}

	private static function readConfig(IBackend $backend): array {
		try {
			$fh = $backend->get('config');
		} catch (BackendException) {
			throw new RepositoryException('No repository at this location');
		}
		$config = json_decode((string)stream_get_contents($fh), true, 512, JSON_THROW_ON_ERROR);
		fclose($fh);
		if (($config['format'] ?? 0) !== self::FORMAT) {
			throw new RepositoryException('Unsupported repository format');
		}
		return $config;
	}

	/** Replace the passphrase-wrapped key slots in this repository's config (after a slot change). */
	public function updateWrappedKey(array $wrappedKey): void {
		$config = self::readConfig($this->backend);
		$config['key'] = $wrappedKey;
		self::putString($this->backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
	}

	public function id(): string {
		return $this->repositoryId;
	}

	public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
		$config = self::readConfig($backend);
		$repo = new self($backend, KeyRing::unwrapAny($config['key'], $passphrase), $config['id']);
		$repo->index->load();
		$repo->loadCatalog(null); // disaster recovery on a new server: no anchor yet, trust the newest valid chain
		return $repo;
	}

	/**
	 * Back up a local directory tree. Files whose size and mtime match the parent snapshot are not
	 * read again; their blob list is reused.
	 *
	 * @return array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int}
	 */
	public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
		$source = rtrim($source, '/');
		$previous = $parent !== null ? $this->loadEntries($parent) : [];
		$packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
		$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];

		$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
		$snapshotId = bin2hex(random_bytes(16));
		fwrite($tree, json_encode(['snapshot' => $snapshotId, 'time' => gmdate('c'), 'parent' => $parent, 'label' => $label, 'source' => $source], JSON_THROW_ON_ERROR) . "\n");

		foreach ($this->walk($source) as $relative => $info) {
			$stats['files']++;
			$prev = $previous[$relative] ?? null;
			if ($prev !== null && $prev['s'] === $info->getSize() && $prev['m'] === $info->getMTime()) {
				$blobs = $prev['b'];
				$stats['reused']++;
			} else {
				$blobs = $this->readStable($info->getPathname(), $packs, $stats);
				$stats['read']++;
			}
			fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
		}
		$packs->flush();

		// The snapshot is written last: it only references blobs whose packs are already stored.
		rewind($tree);
		$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
		$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
		rewind($encrypted);
		$this->backend->put('snapshots/' . $snapshotId, $encrypted);
		fclose($tree);
		$this->catalog->snapshots[$snapshotId] = true;
		$this->catalog->write();

		return ['snapshot' => $snapshotId] + $stats;
	}

	/**
	 * Restore a snapshot (or only paths starting with $prefix) into a local directory. A $prefix
	 * "dir/file/" also restores the file "dir/file" itself.
	 *
	 * @param bool $stripPrefix write "$target/<path without $prefix>" instead of
	 *             "$target/<full logical path>" (disaster recovery: restoring e.g. "data/" or
	 *             "config/" directly into the real data/config directory, not a subdirectory
	 *             named after the prefix). Ignored when $prefix is '' (nothing to strip).
	 * @param array<string, string>|null $skipped salvage: an array to collect the files that could
	 *             not be read or decrypted (path => reason), which are then left out instead of
	 *             stopping the restore; null (default) stops at the first such file
	 * @param bool $keepUnchanged leave a file that is already in place with the size and
	 *             modification time of the snapshot, as a backup takes such a file as unchanged:
	 *             a restore that was stopped and runs again goes on where it was, without reading
	 *             what it already restored from the location again
	 */
	public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null, bool $keepUnchanged = false): int {
		if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) {
			throw new RepositoryException('Invalid path prefix');
		}
		$plan = $this->readAhead($snapshotId, $prefix, $target, $stripPrefix, $keepUnchanged);
		try {
			return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan, $skipped, $keepUnchanged);
		} finally {
			$plan?->close();
			if ($plan !== null) {
				@rmdir($plan->spool);
			}
		}
	}

	/**
	 * Start reading the packs of a restore ahead, when the location can (IReadsAhead): a first
	 * pass over the snapshot lists the ranges of packs it needs, in order. The spool is a
	 * folder in the target, which has room for the restore anyway.
	 */
	private function readAhead(string $snapshotId, string $prefix, string $target, bool $stripPrefix, bool $keepUnchanged): ?ReadAheadPlan {
		if (!$this->backend instanceof IReadsAhead || !$this->backend->canReadAhead()) {
			return null;
		}
		// what a restore that was stopped left behind: its files are unlinked once opened, but one
		// that was being written when the process was killed stays (8 MiB, DR test 2026-11-05)
		$stale = glob(rtrim($target, '/') . '/.ngb-restore-spool-*', GLOB_ONLYDIR);
		foreach ($stale === false ? [] : $stale as $dir) {
			if (is_link($dir) || preg_match('/^\.ngb-restore-spool-[0-9a-f]{8}$/', basename($dir)) !== 1) {
				continue; // not a spool of ours, whatever it is called
			}
			$files = glob($dir . '/*');
			foreach ($files === false ? [] : $files as $file) {
				if (is_file($file) && !is_link($file)) {
					@unlink($file);
				}
			}
			@rmdir($dir);
		}
		$plan = new ReadAheadPlan(rtrim($target, '/') . '/.ngb-restore-spool-' . bin2hex(random_bytes(4)));
		try {
			foreach ($this->streamEntries($snapshotId) as $entry) {
				if (!self::underPrefix($entry['p'], $prefix)) {
					continue;
				}
				if ($keepUnchanged && self::inPlace(self::destination($target, $entry['p'], $prefix, $stripPrefix), $entry)) {
					continue;
				}
				foreach ($entry['b'] as $id) {
					try {
						[$pack, $offset, $length] = $this->index->get($id);
					} catch (RepositoryException) {
						continue; // the restore reports it when it gets there
					}
					$plan->add('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
				}
			}
		} catch (\Throwable) {
			return null; // reading ahead is only faster: the restore itself reads the snapshot again and reports what is wrong
		}
		if (count($plan->ranges()) < 2 || (!is_dir($plan->spool) && !@mkdir($plan->spool, 0700, true))) {
			return null;
		}
		try {
			$streams = $this->backend->readAhead($plan->ranges(), $plan->spool);
		} catch (\Throwable) {
			$streams = null; // the usual reads report what is wrong
		}
		if ($streams === null) {
			@rmdir($plan->spool);
			return null;
		}
		$plan->readFrom($streams);
		return $plan;
	}

	/** @param array<string, string>|null $skipped see restore() */
	private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan, ?array &$skipped, bool $keepUnchanged): int {
		$count = 0;
		$cleaned = [];
		foreach ($this->streamEntries($snapshotId) as $entry) {
			if (!self::underPrefix($entry['p'], $prefix)) {
				continue;
			}
			if ($heartbeat !== null) {
				$heartbeat(); // also for a long row of files that are skipped or have no data
			}
			$dest = self::destination($target, $entry['p'], $prefix, $stripPrefix);
			$dir = self::directoryInTarget($target, dirname($dest), $entry['p']);
			if (is_link($dest)) {
				throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $entry['p']);
			}
			if (!isset($cleaned[$dir])) {
				self::removeStaleTemporaries($dir);
				$cleaned[$dir] = true;
			}
			if ($keepUnchanged && self::inPlace($dest, $entry)) {
				$count++;
				continue;
			}
			// Write next to the destination and rename after all blobs verified: at most one file
			// is ever present twice.
			$tmp = $dest . '.ngb-restore-' . bin2hex(random_bytes(6));
			$out = @fopen($tmp, 'xb'); // fails if anything (including a symlink) already exists there
			if ($out === false) {
				throw new RestoreTargetException('Cannot create a temporary file for ' . $entry['p'] . ' (disk full, or no permission?)');
			}
			try {
				foreach ($entry['b'] as $id) {
					self::writeAll($out, $this->restoreBlob($id, $plan), $entry['p']);
					if ($heartbeat !== null) {
						$heartbeat();
					}
				}
				if (!@fclose($out)) {
					throw new RestoreTargetException('Cannot write ' . $entry['p'] . ' to the restore directory (disk full?)');
				}
				clearstatcache(true, $tmp);
				if (filesize($tmp) !== $entry['s']) {
					throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
				}
			} catch (RepositoryException|BackendException|\OCA\NgBackup\Crypto\CryptoException $e) {
				// What the location gave back for this file is missing or damaged. Writing to the
				// target (below) is not part of this: a full disk must still stop the restore.
				if (is_resource($out)) {
					fclose($out);
				}
				@unlink($tmp);
				if ($skipped === null) {
					throw $e;
				}
				$skipped[$entry['p']] = $e->getMessage();
				continue;
			} catch (\Throwable $e) {
				if (is_resource($out)) {
					@fclose($out);
				}
				@unlink($tmp);
				throw $e;
			}
			if (!rename($tmp, $dest)) {
				@unlink($tmp);
				throw new RestoreTargetException('Cannot put the restored file in place: ' . $entry['p']);
			}
			if (!touch($dest, $entry['m'])) {
				throw new RestoreTargetException('Cannot set the modification time of ' . $entry['p']);
			}
			$count++;
		}
		// Last: a file can only be restored into a folder that exists, and the meta record was
		// read along with the files, so this asks nothing more of the location.
		foreach ($this->emptyDirs($snapshotId) as [$path, $mtime]) {
			if (self::underPrefix($path, $prefix)) {
				if ($heartbeat !== null) {
					$heartbeat();
				}
				// The prefix itself, stripped, is the target (for a file it is the file's name there).
				$dest = $stripPrefix && $path === rtrim($prefix, '/') ? rtrim($target, '/') : self::destination($target, $path, $prefix, $stripPrefix);
				self::restoreEmptyDir($target, $dest, $path, $mtime);
			}
		}
		// Very last, nothing is written in them any more. Also on a run that goes on after a stop, and
		// the deepest first. The time is not worth stopping for: all the files are there by now.
		foreach (array_reverse($this->folderTimes($snapshotId)) as [$path, $mtime]) {
			if (!self::underPrefix($path, $prefix) || ($stripPrefix && $path === rtrim($prefix, '/'))) {
				continue; // the target itself is not ours to change
			}
			if ($heartbeat !== null) {
				$heartbeat();
			}
			$dest = self::destination($target, $path, $prefix, $stripPrefix);
			@touch(self::directoryInTarget($target, $dest, $path), $mtime);
		}
		return $count;
	}

	/**
	 * The folders with something in them that a snapshot recorded, parents before children (none
	 * before 1.1.0). Damage here is no reason to give up on the files of a restore.
	 *
	 * @return list<array{0: string, 1: int}> path, modification time
	 */
	public function folderTimes(string $snapshotId): array {
		$meta = $this->lastMeta[$snapshotId] ?? $this->snapshotMeta($snapshotId);
		$packed = is_string($meta['folderTimes'] ?? null) ? base64_decode($meta['folderTimes'], true) : false;
		$plain = $packed === false ? false : @gzinflate($packed);
		$folders = [];
		foreach ($plain === false ? [] : explode("\n", $plain) as $line) {
			$folder = json_decode($line, true);
			if (is_array($folder) && is_string($folder[0] ?? null) && self::isSafePath($folder[0]) && is_int($folder[1] ?? null)) {
				$folders[] = [$folder[0], $folder[1]];
			}
		}
		return $folders;
	}

	/**
	 * The folders without anything in them that a snapshot recorded (none before 1.0.3).
	 *
	 * @return list<array{0: string, 1: int}> path, modification time
	 */
	public function emptyDirs(string $snapshotId): array {
		$meta = $this->lastMeta[$snapshotId] ?? $this->snapshotMeta($snapshotId);
		$dirs = [];
		foreach (is_array($meta['emptyDirs'] ?? null) ? $meta['emptyDirs'] : [] as $dir) {
			// One folder that makes no sense is no reason to give up on the files of a restore.
			if (is_array($dir) && is_string($dir[0] ?? null) && self::isSafePath($dir[0]) && is_int($dir[1] ?? null)) {
				$dirs[] = [$dir[0], $dir[1]];
			}
		}
		return $dirs;
	}

	private static function restoreEmptyDir(string $target, string $dest, string $path, int $mtime): void {
		if (is_link($dest)) {
			throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $path);
		}
		$existed = is_dir($dest);
		self::directoryInTarget($target, $dest, $path);
		if (!$existed && !@touch($dest, $mtime)) {
			throw new RestoreTargetException('Cannot set the modification time of ' . $path);
		}
	}

	/**
	 * Make sure a folder exists below the target, without following a link out of it: what exists
	 * of the path is checked before anything is created.
	 *
	 * @return string the folder's real path
	 */
	private static function directoryInTarget(string $target, string $dir, string $path): string {
		if (!is_dir($target) && !@mkdir($target, 0700, true) && !is_dir($target)) {
			throw new RestoreTargetException('Cannot create the restore directory ' . $target);
		}
		$base = realpath($target);
		$existing = $dir;
		while (!file_exists($existing) && !is_link($existing) && dirname($existing) !== $existing) {
			$existing = dirname($existing);
		}
		$real = realpath($existing);
		if ($base === false || $real === false || ($real !== $base && !str_starts_with($real, $base . '/'))) {
			throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $path);
		}
		if ($existing === $dir) {
			if (!is_dir($real)) {
				throw new RestoreTargetException('Cannot create the folder for ' . $path . ': a file is in the way');
			}
			return $real;
		}
		if (!@mkdir($dir, 0700, true) && !is_dir($dir)) {
			throw new RestoreTargetException('Cannot create the folder for ' . $path . ' (disk full, no permission, or a file in the way?)');
		}
		return (string)realpath($dir);
	}

	/**
	 * Write all of $data, or stop: a short write leaves a file that looks damaged only by its size.
	 * A write may take only part of the data and still be fine; one that takes nothing is not.
	 *
	 * @param resource $out
	 */
	public static function writeAll($out, string $data, string $path): void {
		while ($data !== '') {
			$written = @fwrite($out, $data);
			if ($written === false || $written === 0) {
				throw new RestoreTargetException('Cannot write ' . $path . ' to the restore directory (disk full?)');
			}
			$data = substr($data, $written);
		}
	}

	/** Where restore() writes the snapshot path $path. */
	private static function destination(string $target, string $path, string $prefix, bool $stripPrefix): string {
		$relPath = $path;
		if ($stripPrefix && $prefix !== '') {
			$relPath = $path === rtrim($prefix, '/') ? basename($path) : substr($path, strlen($prefix));
		}
		return rtrim($target, '/') . '/' . $relPath;
	}

	/**
	 * A regular file (no link) at $dest with the size and modification time of the snapshot
	 * entry: what a backup takes as unchanged, and what a finished restore of it leaves.
	 *
	 * @param array{s: int, m: int, ...} $entry
	 */
	private static function inPlace(string $dest, array $entry): bool {
		clearstatcache(true, $dest);
		return !is_link($dest) && is_file($dest) && filesize($dest) === $entry['s'] && filemtime($dest) === $entry['m'];
	}

	/** $prefix is "" (everything) or ends with "/"; the path without that "/" is a file to restore too. */
	private static function underPrefix(string $path, string $prefix): bool {
		return $prefix === '' || str_starts_with($path, $prefix) || $path === rtrim($prefix, '/');
	}

	/**
	 * Remove what a restore that was stopped left in a directory this restore writes to:
	 * "<name>.ngb-restore-<hex>" from here, ".<name>.ngb-restore-<hex>" from a restore through
	 * Nextcloud. A files:scan would otherwise make them show up in Files.
	 */
	public static function removeStaleTemporaries(string $dir): void {
		$names = scandir($dir);
		foreach ($names === false ? [] : $names as $name) {
			$path = $dir . '/' . $name;
			if (self::isRestoreTemporary($name) && is_file($path) && !is_link($path)) {
				@unlink($path);
			}
		}
	}

	public static function isRestoreTemporary(string $name): bool {
		return preg_match('/.\.ngb-restore-[0-9a-f]{6}(?:[0-9a-f]{6})?$/', $name) === 1;
	}

	/**
	 * Start writing a generated stream (e.g. a table dump) as blobs. Call flushPacks() when the
	 * run is done, so the last pack is uploaded before anything references it.
	 */
	public function blobWriter(array &$stats): BlobWriter {
		$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
		$packs = $this->runPacks;
		return new BlobWriter(function (string $data) use ($packs, &$stats): string {
			return $this->storeBlob($data, $packs, $stats);
		});
	}

	/** See BlobIndex::holdCache(): no database writes until releaseIndexCache(). */
	public function holdIndexCache(): void {
		$this->index->holdCache();
	}

	public function releaseIndexCache(): void {
		$this->index->releaseCache();
	}

	public function flushPacks(): void {
		$this->runPacks?->flush();
	}

	/**
	 * Read a blob stream back as lines (lines may span blob boundaries).
	 *
	 * @param list<string> $blobIds
	 * @return \Generator<string>
	 */
	public function readLines(array $blobIds): \Generator {
		$carry = '';
		foreach ($this->loadBlobs($blobIds) as $blob) {
			$data = $carry . $blob;
			$lines = explode("\n", $data);
			$carry = array_pop($lines);
			yield from $lines;
		}
		if ($carry !== '') {
			yield $carry;
		}
	}

	/** Store one blob of file data in the current run (deduplicated); returns its id. */
	public function storeData(string $data, array &$stats): string {
		$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
		return $this->storeBlob($data, $this->runPacks, $stats);
	}

	/**
	 * Write a snapshot from a meta record and the tree lines (JSON, one per file). Must be called
	 * after flushPacks(), so the snapshot only references stored blobs.
	 *
	 * @param iterable<string> $lines
	 */
	public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void {
		$this->putSnapshot($snapshotId, $meta, $lines);
		$this->recordSnapshot($snapshotId);
	}

	/**
	 * Store a snapshot object without touching the catalog. Use recordSnapshot() afterwards, from a
	 * repository opened inside the catalog lock, so the catalog write builds on the latest generation.
	 *
	 * @param iterable<string> $lines
	 */
	public function putSnapshot(string $snapshotId, array $meta, iterable $lines): void {
		$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
		fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n");
		foreach ($lines as $line) {
			if ($line !== '') {
				fwrite($tree, $line . "\n");
			}
		}
		rewind($tree);
		$encrypted = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
		$this->cipher->encrypt($tree, $encrypted, 'snapshot:' . $snapshotId);
		fclose($tree);
		rewind($encrypted);
		$this->backend->put('snapshots/' . $snapshotId, $encrypted);
	}

	public function recordSnapshot(string $snapshotId): void {
		$this->catalog->snapshots[$snapshotId] = true;
		$this->catalog->write();
	}

	/** @return array<string, array{s:int, m:int, b:list<string>}> */
	public function snapshotEntries(string $snapshotId): array {
		return $this->loadEntries($snapshotId);
	}

	/**
	 * @param list<string> $blobIds
	 * @return \Generator<string> verified blob contents, in order
	 */
	public function readBlobs(array $blobIds): \Generator {
		foreach ($blobIds as $id) {
			yield $this->loadBlob($id);
		}
	}

	/** Relative path of non-empty segments, without ".", "..", NUL, backslash or control characters. */
	public static function isSafePath(string $path): bool {
		if ($path === '' || str_starts_with($path, '/') || preg_match('/[\x00-\x1f\\\\]/', $path)) {
			return false;
		}
		foreach (explode('/', $path) as $segment) {
			if ($segment === '' || $segment === '.' || $segment === '..') {
				return false;
			}
		}
		return true;
	}

	/** Small encrypted objects (manifests). */
	public function putObject(string $path, string $data): void {
		self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path));
	}

	public function getObject(string $path): string {
		$fh = $this->backend->get($path);
		$data = $this->cipher->decryptString((string)stream_get_contents($fh), 'object:' . $path);
		fclose($fh);
		return $data;
	}

	/**
	 * Move a snapshot (and its database dump) to the trash on the location. Its data stays in
	 * use until purgeTrash() removes it after the deletion delay, so a malicious or mistaken
	 * cleanup can be undone with untrash() within that time.
	 */
	public function forget(string $snapshotId, string $by = ''): void {
		$meta = $this->snapshotMeta($snapshotId);
		$dbInTrash = null;
		if (isset($meta['db']) && is_string($meta['db']) && $this->backend->exists($meta['db'])) {
			// Its encryption is bound to its path (object:db/...), which changes on a move, so it
			// must be decrypted and re-encrypted under the new path, not moved as raw bytes.
			$dbInTrash = 'trash/' . $meta['db'];
			$this->putObject($dbInTrash, $this->getObject($meta['db']));
		}
		// Copy first, switch the catalog, delete the originals last: a crash at any point leaves
		// either the old state or the new one plus unreferenced leftovers, never a catalog entry
		// pointing at a missing object. The snapshot object is bound to its id (not its path),
		// so it can be copied as raw bytes.
		$this->copyRaw('snapshots/' . $snapshotId, 'trash/snapshots/' . $snapshotId);
		$this->putObject('trash/info/' . $snapshotId, json_encode(['forgottenAt' => time(), 'by' => $by, 'db' => $meta['db'] ?? null,
			'dbInTrash' => $dbInTrash, 'time' => $meta['time'] ?? null, 'label' => $meta['label'] ?? ''], JSON_THROW_ON_ERROR));
		unset($this->catalog->snapshots[$snapshotId]);
		$this->catalog->trash[$snapshotId] = true;
		$this->catalog->write();
		if ($dbInTrash !== null) {
			$this->backend->delete($meta['db']);
		}
		$this->backend->delete('snapshots/' . $snapshotId);
	}

	/** Undo forget(): move a snapshot back from the trash. */
	public function untrash(string $snapshotId): void {
		$info = $this->trashInfo($snapshotId);
		$restoreDb = $info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash']);
		if ($restoreDb) {
			$this->putObject($info['db'], $this->getObject($info['dbInTrash']));
		}
		// Same order as forget(): copy, switch the catalog, then delete the trash copies.
		$this->copyRaw('trash/snapshots/' . $snapshotId, 'snapshots/' . $snapshotId);
		$this->catalog->snapshots[$snapshotId] = true;
		unset($this->catalog->trash[$snapshotId]);
		$this->catalog->write();
		if ($restoreDb) {
			$this->backend->delete($info['dbInTrash']);
		}
		$this->backend->delete('trash/snapshots/' . $snapshotId);
		$this->backend->delete('trash/info/' . $snapshotId);
	}

	/** Copy an object as raw bytes, buffered locally (remote read streams cannot be uploaded directly everywhere). */
	private function copyRaw(string $from, string $to): void {
		$in = $this->backend->get($from);
		$buffer = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
		if ($buffer === false) {
			fclose($in);
			throw new RepositoryException("Cannot buffer $from");
		}
		try {
			if (stream_copy_to_stream($in, $buffer) === false) {
				throw new RepositoryException("Cannot read $from");
			}
		} finally {
			fclose($in);
		}
		rewind($buffer);
		$this->backend->put($to, $buffer);
	}

	/** Record a user_migration export's manifest in the catalog (rollback-protected like a snapshot). */
	public function recordUserExport(string $manifestPath): void {
		$this->catalog->userExports[$manifestPath] = true;
		$this->catalog->write();
	}

	/**
	 * User export manifests according to the verified catalog, newest first. A manifest the
	 * catalog lists but the location no longer has means it was deleted or hidden behind NG
	 * Backup's back, same as a missing snapshot (see listSnapshots()).
	 *
	 * @return list<string>
	 */
	public function listUserExports(?string $uid = null): array {
		$prefix = $uid !== null ? 'users/' . $uid . '/' : 'users/';
		$out = [];
		foreach (array_keys($this->catalog->userExports) as $path) {
			if (!str_starts_with($path, $prefix)) {
				continue;
			}
			if (!$this->backend->exists($path)) {
				throw new RepositoryException("User export $path is missing on the location (deleted or hidden outside NG Backup)");
			}
			$out[] = $path;
		}
		sort($out, SORT_STRING);
		return array_reverse($out);
	}

	/** Whether $manifestPath is a user export the verified catalog actually recorded. */
	public function isRecordedUserExport(string $manifestPath): bool {
		return isset($this->catalog->userExports[$manifestPath]);
	}

	/**
	 * The recorded export $ref names: a manifest path (users/<uid>/<exportId>) as is, or a bare
	 * user id (also "users/<uid>") when that user has exactly one export here. Anything else
	 * throws, with the exports to choose from in the message.
	 */
	public function resolveUserExport(string $ref): string {
		if ($this->isRecordedUserExport($ref)) {
			return $ref;
		}
		$rest = trim($ref, '/');
		if (str_starts_with($rest, 'users/')) {
			$rest = substr($rest, strlen('users/'));
		}
		$uid = explode('/', $rest)[0];
		$exports = $uid !== '' ? $this->listUserExports($uid) : [];
		if (!str_contains($rest, '/') && count($exports) === 1) {
			return $exports[0];
		}
		if ($exports === []) {
			throw new RepositoryException("$ref is not a recorded user export for this location, and $uid has no exports here. Expected users/<uid>/<exportId>; occ backup:user:list shows the exports");
		}
		$which = str_contains($rest, '/') ? "$ref is not a recorded user export for this location" : "$uid has " . count($exports) . ' exports here';
		throw new RepositoryException("$which; pass one of these (newest first):\n  " . implode("\n  ", $exports));
	}

	/** Permanently delete a user export. Its blobs are freed by the next prune() like any other. */
	public function forgetUserExport(string $manifestPath): void {
		if (!isset($this->catalog->userExports[$manifestPath])) {
			throw new RepositoryException("$manifestPath is not a recorded user export");
		}
		unset($this->catalog->userExports[$manifestPath]);
		$this->catalog->write();
		$this->backend->delete($manifestPath);
	}

	/** @return array<string, array{forgottenAt:int, by:string, db:?string, dbInTrash:?string, time:?string, label:string}> */
	public function trash(): array {
		$out = [];
		foreach (array_keys($this->catalog->trash) as $id) {
			try {
				$out[$id] = $this->trashInfo($id);
			} catch (\Throwable) {
				throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)");
			}
		}
		return $out;
	}

	/** Permanently delete trash entries older than $delaySeconds. @return list<string> purged snapshot ids */
	public function purgeTrash(int $delaySeconds): array {
		$doomed = [];
		foreach ($this->trash() as $id => $info) {
			if (time() - $info['forgottenAt'] < $delaySeconds) {
				continue;
			}
			$doomed[$id] = $info;
			unset($this->catalog->trash[$id]);
		}
		if ($doomed === []) {
			return [];
		}
		$this->catalog->write();
		foreach ($doomed as $id => $info) {
			if ($info['dbInTrash'] !== null) {
				$this->backend->delete($info['dbInTrash']);
			}
			$this->backend->delete('trash/snapshots/' . $id);
			$this->backend->delete('trash/info/' . $id);
		}
		return array_keys($doomed);
	}

	/**
	 * Part of trash(), for a page that shows how far it is: each entry is a read of its own on the
	 * location. Reads the entries with an id after $after, in order, until $deadline
	 * (microtime(true)), at least one. The id, not a position: a cleanup that changes the trash
	 * between two parts then makes no entry come twice or be missed.
	 *
	 * @return array{total: int, items: array<string, array<string, mixed>>}
	 */
	public function trashPage(string $after, float $deadline): array {
		$ids = array_map('strval', array_keys($this->catalog->trash));
		sort($ids, SORT_STRING);
		$items = [];
		foreach ($ids as $id) {
			if (strcmp($id, $after) <= 0) {
				continue;
			}
			try {
				$items[$id] = $this->trashInfo($id);
			} catch (\Throwable) {
				throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)");
			}
			if (microtime(true) >= $deadline) {
				break;
			}
		}
		return ['total' => count($ids), 'items' => $items];
	}

	private function trashInfo(string $snapshotId): array {
		return json_decode($this->getObject('trash/info/' . $snapshotId), true, 512, JSON_THROW_ON_ERROR);
	}

	/**
	 * Delete packs no snapshot or user export uses any more, and repack packs where more than
	 * $repackThreshold of the bytes are unused. Must not run while a backup to this repository
	 * is in progress (its blobs are not referenced by a snapshot yet); callers hold the lock.
	 *
	 * @return array{packs:int, deleted:int, repacked:int, freedBytes:int, keptBlobs:int, unusedBlobs:int}
	 */
	public function prune(bool $dryRun = false, float $repackThreshold = 0.5, array $ignoreSnapshots = []): array {
		// Deleting is decided on the authenticated index files on the location, never on the local cache.
		$this->index->load(true);
		$used = [];
		$ignore = array_flip($ignoreSnapshots); // e.g. snapshots a dry run would forget first
		foreach ($this->listSnapshots() as $sid) {
			if (isset($ignore[$sid])) {
				continue;
			}
			foreach ($this->streamEntries($sid) as $e) {
				foreach ($e['b'] as $b) {
					$used[$b] = true;
				}
			}
			$meta = $this->snapshotMeta($sid);
			if (isset($meta['db']) && $this->backend->exists($meta['db'])) {
				foreach (json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
					foreach ($t['blobs'] as $b) {
						$used[$b] = true;
					}
				}
			}
		}
		// Snapshots in the trash keep their data until purgeTrash() removed them.
		foreach ($this->trash() as $sid => $info) {
			foreach ($this->streamEntries($sid, 'trash/snapshots/') as $e) {
				foreach ($e['b'] as $b) {
					$used[$b] = true;
				}
			}
			if ($info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash'])) {
				foreach (json_decode($this->getObject($info['dbInTrash']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
					foreach ($t['blobs'] as $b) {
						$used[$b] = true;
					}
				}
			}
		}
		// listUserExports() throws when a recorded export's manifest is missing: then stop, since
		// freeing its blobs would turn a tampering or storage alarm into irreversible deletion.
		$exports = array_unique(array_merge($this->listUserExports(), $this->backend->list('users')));
		foreach ($exports as $path) {
			foreach (json_decode($this->getObject($path), true, 512, JSON_THROW_ON_ERROR)['entries'] as $e) {
				foreach ($e['b'] ?? [] as $b) {
					$used[$b] = true;
				}
			}
		}

		$stats = ['packs' => 0, 'deleted' => 0, 'repacked' => 0, 'freedBytes' => 0, 'keptBlobs' => count($used), 'unusedBlobs' => 0];
		$packs = $this->index->packs();
		$stats['packs'] = count($packs);
		$writer = null;
		$toRemove = [];
		foreach ($packs as $packId => $blobs) {
			$total = 0;
			$unused = 0;
			foreach ($blobs as $b) {
				$total += $b['length'];
				if (!isset($used[$b['id']])) {
					$unused += $b['length'];
					$stats['unusedBlobs']++;
				}
			}
			if ($unused === 0 || ($unused < $total && $unused / $total <= $repackThreshold)) {
				continue;
			}
			$stats['freedBytes'] += $unused;
			if ($unused === $total) {
				$stats['deleted']++;
			} else {
				$stats['repacked']++;
				if (!$dryRun) {
					// No size limit here: repacking is how space is freed.
					$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
					$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
					foreach ($blobs as $b) {
						if (isset($used[$b['id']])) {
							[, , , $raw, $flags] = $this->index->get($b['id']);
							$writer->addRaw($b['id'], $this->backend->getRange($path, $b['offset'], $b['length']), $raw, $flags);
						}
					}
				}
			}
			$toRemove[] = $packId;
		}
		if ($dryRun) {
			return $stats;
		}
		// New packs and their index entries are durable before any old pack is removed.
		$writer?->flush();
		foreach ($toRemove as $packId) {
			$this->index->removePack($packId);
			$this->backend->delete('packs/' . substr($packId, 0, 2) . '/' . $packId);
		}
		return $stats;
	}

	/**
	 * Snapshots according to the verified catalog. A snapshot the catalog lists but the location
	 * no longer has means it was deleted or hidden behind NG Backup's back.
	 *
	 * @return list<string>
	 */
	public function listSnapshots(): array {
		$present = array_flip(array_map('basename', $this->backend->listFiles('snapshots')));
		$listed = array_keys($this->catalog->snapshots);
		foreach ($listed as $id) {
			if (!isset($present[$id])) {
				throw new RepositoryException("Snapshot $id is missing on the location (deleted or hidden outside NG Backup)");
			}
		}
		sort($listed);
		return $listed;
	}

	public function blobCount(): int {
		return $this->index->count();
	}

	/**
	 * Read a file into blobs and make sure it did not change while reading (size, mtime, ctime
	 * before and after). A file that keeps changing is an error rather than a torn backup.
	 *
	 * @return list<string>
	 */
	private function readStable(string $path, PackWriter $packs, array &$stats): array {
		for ($attempt = 1; $attempt <= 3; $attempt++) {
			$before = self::fileVersion($path);
			$blobs = [];
			$fh = fopen($path, 'rb');
			while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
				$blobs[] = $this->storeBlob($data, $packs, $stats);
			}
			fclose($fh);
			if (self::fileVersion($path) === $before) {
				return $blobs;
			}
			$stats['retried'] = ($stats['retried'] ?? 0) + 1;
		}
		throw new RepositoryException('File keeps changing while being read: ' . $path);
	}

	/** Cheap file identity: changes when the content may have changed. */
	public static function fileVersion(string $path): string {
		clearstatcache(true, $path);
		$st = @stat($path);
		return $st === false ? 'missing' : implode(':', [$st['size'], $st['mtime'], $st['ctime'], $st['ino']]);
	}

	private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
		$id = $this->keys->blobId($data);
		if ($this->index->has($id)) {
			$stats['dupBlobs'] = ($stats['dupBlobs'] ?? 0) + 1;
			return $id;
		}
		$flags = 0;
		$payload = $data;
		$deflated = self::worthDeflating($data) ? gzdeflate($data, 6) : false;
		if ($deflated !== false && strlen($deflated) < strlen($data) * 0.95) {
			$payload = $deflated;
			$flags |= self::FLAG_DEFLATE;
		}
		$stats['uploaded'] = ($stats['uploaded'] ?? 0) + $packs->add($id, $this->cipher->encryptString($payload, 'blob:' . $id), strlen($data), $flags);
		$stats['newBlobs'] = ($stats['newBlobs'] ?? 0) + 1;
		return $id;
	}

	/**
	 * Photos, video and archives are compressed already: deflating them takes most of a backup's
	 * CPU time (about 24 MiB/s) only to be thrown away, as it saves less than the 5% that keeps
	 * it. A fast deflate of three small slices tells first; below 10% savings there, the full
	 * deflate is skipped.
	 */
	private static function worthDeflating(string $data): bool {
		$n = strlen($data);
		if ($n < self::DEFLATE_PROBE_FROM) {
			return true;
		}
		$k = self::DEFLATE_PROBE_SLICE;
		$probe = substr($data, 0, $k) . substr($data, intdiv($n - $k, 2), $k) . substr($data, $n - $k);
		$deflated = gzdeflate($probe, 1);
		return $deflated !== false && strlen($deflated) * 10 < strlen($probe) * 9;
	}

	/**
	 * Blobs spread over many packs (the tree segments of a run, one per step), through
	 * IBackend::getRanges(): on a remote location one at a time is a round trip each, which for
	 * a long run made writing its snapshot take a quarter of an hour. Blobs that follow each
	 * other in a pack are read as one range.
	 *
	 * @param list<string> $ids
	 * @return \Generator<int, string> verified blob contents, in order
	 */
	private function loadBlobs(array $ids): \Generator {
		/** @var list<array{0:string, 1:int, 2:int}> $ranges [path, offset, length] of each read */
		$ranges = [];
		/** @var list<list<array{0:string, 1:int, 2:int}>> $parts per read: [blob id, offset in the read, length] */
		$parts = [];
		// the read being extended
		$curPath = null;
		$curOffset = 0;
		$curLength = 0;
		$curParts = [];
		foreach ($ids as $id) {
			[$pack, $offset, $length] = $this->index->get($id);
			$path = 'packs/' . substr($pack, 0, 2) . '/' . $pack;
			if ($curPath === $path && $curOffset + $curLength === $offset && $curLength + $length <= self::MERGED_READ_MAX) {
				$curParts[] = [$id, $curLength, $length];
				$curLength += $length;
				continue;
			}
			if ($curPath !== null) {
				$ranges[] = [$curPath, $curOffset, $curLength];
				$parts[] = $curParts;
			}
			[$curPath, $curOffset, $curLength, $curParts] = [$path, $offset, $length, [[$id, 0, $length]]];
		}
		if ($curPath !== null) {
			$ranges[] = [$curPath, $curOffset, $curLength];
			$parts[] = $curParts;
		}
		$done = 0;
		try {
			foreach ($this->backend->getRanges($ranges) as $i => $data) {
				foreach ($parts[$i] as [$id, $at, $length]) {
					try {
						$blob = $this->decodeBlob($id, substr($data, $at, $length));
					} catch (\Exception) {
						// a read that went wrong on the way (see loadBlob()): this blob on its own
						$blob = $this->loadBlob($id);
						// its pack stream is closed before the next range is read on the same connection
						$this->packReader->close();
					}
					yield $blob;
					$done++;
				}
			}
		} catch (BackendException) {
			// A pack moved, or the read failed for a while (see loadBlob()): the rest one blob at a
			// time, each read once more after a failure.
			foreach (array_slice($ids, $done) as $id) {
				yield $this->loadBlob($id);
			}
		}
	}

	/** A blob for restore(): read ahead when it could be, otherwise (or when that copy is bad) the usual way. */
	private function restoreBlob(string $id, ?ReadAheadPlan $plan): string {
		if ($plan !== null) {
			[$pack, $offset, $length] = $this->index->get($id);
			$encrypted = $plan->read('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
			if ($encrypted !== null) {
				try {
					return $this->decodeBlob($id, $encrypted);
				} catch (\Exception) {
					// read again from the location below, which reports what is wrong
				}
			}
		}
		return $this->loadBlob($id);
	}

	private function loadBlob(string $id): string {
		[$pack, $offset, $length] = $this->index->get($id);
		try {
			$encrypted = $this->packReader->read($pack, $offset, $length);
		} catch (BackendException $e) {
			// Another server's cleanup may have repacked this blob into a new pack since the index
			// was loaded (a disaster recovery test while the original server still runs): add the
			// new index files, and read the blob where it is now. Or the read failed for a while
			// (a connection that died; the backend has replaced it by now): then read it once
			// more where it was. A second failure is the real one.
			try {
				$this->index->refresh();
			} catch (\Throwable) {
				// the location could not be listed either: read again where the blob was
			}
			[$moved, $offset, $length] = $this->index->get($id);
			try {
				$encrypted = $this->packReader->read($moved, $offset, $length);
			} catch (BackendException) {
				throw $e;
			}
			$pack = $moved;
		}
		try {
			return $this->decodeBlob($id, $encrypted);
		} catch (\Exception $damaged) {
			// Bytes that do not decrypt: damaged on the location, or a read that went wrong on the
			// way (a stream that lost its place: an SMB restore stopped after 17 hours like this).
			// Read exactly this blob once more, on a request of its own; only a second failure
			// means the data itself is damaged.
			$this->packReader->close();
			try {
				$again = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
			} catch (\Exception) {
				throw $damaged;
			}
			$data = $this->decodeBlob($id, $again);
			if (function_exists('OCP\Log\logger')) {
				\OCP\Log\logger('ng_backup')->warning('NG Backup: blob ' . $id . ' did not decrypt as first read from pack ' . $pack . ' at ' . $offset . '; read again, it is fine. First read: ' . self::howItDiffered($encrypted, $again));
			}
			return $data;
		}
	}

	/**
	 * What a read that went wrong looked like next to the right bytes, for the log: where it starts
	 * to differ, and whether the rest is zeros or the right bytes from another place (a stream that
	 * lost its place).
	 */
	private static function howItDiffered(string $wrong, string $right): string {
		$at = strspn($wrong ^ $right, "\0");
		$text = strlen($wrong) . ' bytes, differ from byte ' . $at . ' of ' . strlen($right);
		$tail = substr($wrong, $at, 64);
		if ($tail === '') {
			return $text . ' (too short)';
		}
		if (trim($tail, "\0") === '') {
			return $text . ' (zeros)';
		}
		$found = strpos($right, $tail);
		return $text . ($found !== false ? ' (the right bytes from ' . ($found - $at) . ' bytes further on)' : ' (other bytes)');
	}

	/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */
	private function decodeBlob(string $id, string $encrypted): string {
		[, , , $raw, $flags] = $this->index->get($id);
		$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
		$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
		if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
			throw new RepositoryException('Blob verification failed: ' . $id);
		}
		return $data;
	}

	/** @return array<string, array{s:int, m:int, b:list<string>}> */
	private function loadEntries(string $snapshotId): array {
		$map = [];
		foreach ($this->streamEntries($snapshotId) as $e) {
			$map[$e['p']] = ['s' => $e['s'], 'm' => $e['m'], 'b' => $e['b']];
		}
		return $map;
	}

	/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> entries of a snapshot, in stored order */
	public function entries(string $snapshotId): \Generator {
		return $this->streamEntries($snapshotId);
	}

	/**
	 * Checksum audit of a snapshot: every blob it references (files and the database dump, when
	 * the snapshot has one) must be in the index, freshly re-downloaded and re-authenticated from
	 * the location rather than trusted from the local cache (same reasoning as prune()), and its
	 * pack must still be present. With $deep, each blob is also downloaded and decrypted, which
	 * verifies its AEAD authentication tag and its content hash against the blob id (loadBlob()
	 * already does both on every real read -- this just does it for everything in the snapshot,
	 * deliberately, rather than relying on it to come up during a later restore). A blob
	 * referenced by more than one file is only checked once; a missing database manifest is a
	 * failure, not something to silently skip over.
	 *
	 * @param ?callable(array{phase:string, done:int, total:?int, blobs?:int, bytes?:int}):void $heartbeat
	 *        called often during the check, with how far it is: phase "index" (index files
	 *        downloaded), "packs" (pack folders listed), "files" and "database" (files of the
	 *        snapshot checked); callers use it to keep a time-boxed lease alive and to report
	 * @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
	 */
	public function verify(string $snapshotId, bool $deep = false, ?callable $heartbeat = null): array {
		$heartbeat ??= static function (array $progress): void {
		};
		$meta = $this->snapshotMeta($snapshotId);
		$filesTotal = isset($meta['stats']['files']) ? (int)$meta['stats']['files'] : null;
		$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
			$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
		});
		$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
		$seen = [];
		[$packSeen, $listedDirs] = $this->presentPacks($heartbeat);
		foreach ($this->streamEntries($snapshotId) as $entry) {
			$stats['filesChecked']++;
			foreach ($entry['b'] as $id) {
				if (isset($seen[$id])) {
					continue;
				}
				$seen[$id] = true;
				$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
				$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
					'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
			}
		}
		if (isset($meta['db']) && is_string($meta['db'])) {
			if (!$this->backend->exists($meta['db'])) {
				$stats['missing'][] = $meta['db'];
			} else {
				$dbManifest = json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
				foreach ($dbManifest['tables'] as $info) {
					foreach ($info['blobs'] as $id) {
						if (isset($seen[$id])) {
							continue;
						}
						$seen[$id] = true;
						$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
						$heartbeat(['phase' => 'database', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
							'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
					}
				}
			}
		}
		return $stats;
	}

	/**
	 * Which packs are on the location, from one listing per pack folder instead of a request per
	 * pack. A folder that cannot be listed, or comes back empty (which a storage also reports when
	 * it could not look), is left out: its packs are then checked one by one. With no more packs
	 * than folders nothing is listed: a request per pack is fewer requests then, and on FTP every
	 * request takes a data connection, of which a server may have only a few.
	 *
	 * @return array{0: array<string, bool>, 1: array<string, true>} pack id => present, and the
	 *                                                              folders that were listed
	 */
	private function presentPacks(callable $heartbeat): array {
		$present = [];
		$listed = [];
		if ($this->index->packCount() <= $this->listPacksAbove) {
			return [$present, $listed];
		}
		for ($i = 0; $i < 256; $i++) {
			$dir = sprintf('%02x', $i);
			try {
				$files = $this->backend->listFiles('packs/' . $dir);
			} catch (BackendException) {
				$files = [];
			}
			foreach ($files as $path) {
				$present[basename($path)] = true;
			}
			if ($files !== []) {
				$listed[$dir] = true;
			}
			$heartbeat(['phase' => 'packs', 'done' => $i + 1, 'total' => 256]);
		}
		return [$present, $listed];
	}

	/**
	 * Same checks as verify(), for a user export: every blob its manifest references must be in
	 * the freshly re-authenticated index with its pack present; $deep also decrypts each.
	 *
	 * @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
	 */
	public function verifyUserExport(string $manifestPath, bool $deep = false, ?callable $heartbeat = null): array {
		if (!$this->isRecordedUserExport($manifestPath)) {
			throw new RepositoryException("$manifestPath is not a recorded user export for this location");
		}
		$heartbeat ??= static function (array $progress): void {
		};
		$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
			$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
		});
		$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
		$seen = [];
		$packSeen = [];
		$manifest = json_decode($this->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
		foreach ($manifest['entries'] as $entry) {
			if (($entry['t'] ?? '') !== 'f') {
				continue;
			}
			$stats['filesChecked']++;
			foreach ($entry['b'] ?? [] as $id) {
				if (isset($seen[$id])) {
					continue;
				}
				$seen[$id] = true;
				$this->verifyBlob($id, $deep, $stats, $packSeen);
				$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => null,
					'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
			}
		}
		return $stats;
	}

	/**
	 * @param array<string, bool> $packSeen pack id => present; filled in as packs are checked
	 * @param array<string, true> $listedDirs pack folders whose listing is in $packSeen: a pack
	 *                                        not in it is not there
	 */
	private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen, array $listedDirs = []): void {
		if (!$this->index->has($id)) {
			$stats['missing'][] = $id;
			return;
		}
		[$pack] = $this->index->get($id);
		$packExists = $packSeen[$pack] ??= isset($listedDirs[substr($pack, 0, 2)])
			? false
			: $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
		if (!$packExists) {
			$stats['missing'][] = $id;
			return;
		}
		$stats['blobsChecked']++;
		if (!$deep) {
			return;
		}
		// Reading and checking are kept apart: a read that fails is usually a network or
		// storage error, so it is retried and reported as unreadable (a pack that stays
		// unreadable may still be truncated). Only a blob that was read but does not
		// decrypt or match its id counts as failed: that is proof of damage.
		[, $offset, $length] = $this->index->get($id);
		$encrypted = null;
		for ($attempt = 1; $attempt <= self::VERIFY_READ_ATTEMPTS && $encrypted === null; $attempt++) {
			try {
				$encrypted = $this->packReader->read($pack, $offset, $length);
			} catch (\Exception $e) { // storage adapters throw their own exceptions; \Error (a bug) is not retried
				$stats['readError'] = $e->getMessage();
			}
		}
		if ($encrypted === null) {
			$stats['unreadable'][] = $id;
			return;
		}
		try {
			$stats['bytesChecked'] += strlen($this->decodeBlob($id, $encrypted));
		} catch (\Exception) { // damage; an \Error is a bug and must not be reported as damage
			$stats['failed'][] = $id;
		}
	}

	/** The meta record (first line) of a snapshot. */
	public function snapshotMeta(string $snapshotId): array {
		$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
		$fh = $this->backend->get('snapshots/' . $snapshotId);
		$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
		fclose($fh);
		rewind($plain);
		$meta = json_decode((string)fgets($plain), true, 512, JSON_THROW_ON_ERROR);
		fclose($plain);
		return $meta;
	}

	/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
	private function streamEntries(string $snapshotId, string $dir = 'snapshots/'): \Generator {
		$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
		$fh = $this->backend->get($dir . $snapshotId);
		$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
		fclose($fh);
		rewind($plain);
		$meta = json_decode((string)fgets($plain), true);
		if (is_array($meta)) {
			$this->lastMeta = [$snapshotId => $meta]; // a snapshot never changes
		}
		while (($line = fgets($plain)) !== false) {
			$entry = json_decode($line, true, 512, JSON_THROW_ON_ERROR);
			if (!is_array($entry) || !is_string($entry['p'] ?? null) || !self::isSafePath($entry['p'])
				|| !is_int($entry['s'] ?? null) || !is_array($entry['b'] ?? null)) {
				throw new RepositoryException('Invalid entry in snapshot ' . $snapshotId);
			}
			yield $entry;
		}
		fclose($plain);
	}

	/** @return \Generator<string, \SplFileInfo> relative path => file, in a stable order */
	private function walk(string $root): \Generator {
		$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($root, \FilesystemIterator::SKIP_DOTS));
		$files = [];
		foreach ($it as $file) {
			if ($file->isFile()) {
				$files[substr($file->getPathname(), strlen($root) + 1)] = $file;
			}
		}
		ksort($files, SORT_STRING);
		yield from $files;
	}

	private static function putString(IBackend $backend, string $path, string $data): void {
		$s = fopen('php://memory', 'w+b');
		fwrite($s, $data);
		rewind($s);
		$backend->put($path, $s);
	}

	private static function readFull($fh, int $length): string {
		$buffer = '';
		while (strlen($buffer) < $length && !feof($fh)) {
			$buffer .= (string)fread($fh, $length - strlen($buffer));
		}
		return $buffer;
	}
}