Full restore over SFTP: read packs ahead on four connections at once

A restore read one pack at a time over one SFTP connection, which over a long line is bound by the round trip and by what one TCP connection gets: about 2-3 MiB/s from a Hetzner Storage Box to Canada, with PHP mostly waiting. Repository::restore() now first lists the ranges of packs the restore needs, in order (ReadAheadPlan), and the location reads them ahead into unlinked files in a spool folder in the target (IReadsAhead). Over SFTP, SftpRangeFetcher does that on four connections of its own, 16 requests in flight each, taking the answers of whichever connection has one. A range that fails, a damaged copy, or a blob that moved since the plan is read the usual way, which reports what is wrong. Other locations restore as before. On a test line with 120 ms RTT and 0.2% loss, a 1.6 GiB restore went from 5.1 to 17.8 MiB/s, all files identical.
2026-10-19 01:40:49 +02:00
parent b789d17636
commit ccbbf01c54
12 changed files with 1228 additions and 5 deletions
@@ -9,6 +9,7 @@ namespace OCA\NgBackup\Repository;

 use OCA\NgBackup\Backend\BackendException;
 use OCA\NgBackup\Backend\IBackend;
+use OCA\NgBackup\Backend\IReadsAhead;
 use OCA\NgBackup\Crypto\KeyRing;
 use OCA\NgBackup\Crypto\StreamCipher;

@@ -216,6 +217,53 @@ final class Repository {
 		if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) {
 			throw new RepositoryException('Invalid path prefix');
 		}
+		$plan = $this->readAhead($snapshotId, $prefix, $target);
+		try {
+			return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan);
+		} finally {
+			$plan?->close();
+			if ($plan !== null) {
+				@rmdir($plan->spool);
+			}
+		}
+	}
+
+	/**
+	 * Start reading the packs of a restore ahead, when the location can (IReadsAhead): a first
+	 * pass over the snapshot lists the ranges of packs it needs, in order. The spool is a
+	 * folder in the target, which has room for the restore anyway.
+	 */
+	private function readAhead(string $snapshotId, string $prefix, string $target): ?ReadAheadPlan {
+		if (!$this->backend instanceof IReadsAhead || !$this->backend->canReadAhead()) {
+			return null;
+		}
+		$plan = new ReadAheadPlan(rtrim($target, '/') . '/.ngb-restore-spool-' . bin2hex(random_bytes(4)));
+		foreach ($this->streamEntries($snapshotId) as $entry) {
+			if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
+				continue;
+			}
+			foreach ($entry['b'] as $id) {
+				try {
+					[$pack, $offset, $length] = $this->index->get($id);
+				} catch (RepositoryException) {
+					continue; // the restore reports it when it gets there
+				}
+				$plan->add('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
+			}
+		}
+		if (count($plan->ranges()) < 2 || (!is_dir($plan->spool) && !@mkdir($plan->spool, 0700, true))) {
+			return null;
+		}
+		$streams = $this->backend->readAhead($plan->ranges(), $plan->spool);
+		if ($streams === null) {
+			@rmdir($plan->spool);
+			return null;
+		}
+		$plan->readFrom($streams);
+		return $plan;
+	}
+
+	private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan): int {
 		$count = 0;
 		foreach ($this->streamEntries($snapshotId) as $entry) {
 			if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
@@ -240,7 +288,7 @@ final class Repository {
 				throw new RepositoryException('Cannot create a temporary file for ' . $entry['p']);
 			}
 			foreach ($entry['b'] as $id) {
-				fwrite($out, $this->loadBlob($id));
+				fwrite($out, $this->restoreBlob($id, $plan));
 				if ($heartbeat !== null) {
 					$heartbeat();
 				}
@@ -789,6 +837,22 @@ final class Repository {
 		}
 	}

+	/** A blob for restore(): read ahead when it could be, otherwise (or when that copy is bad) the usual way. */
+	private function restoreBlob(string $id, ?ReadAheadPlan $plan): string {
+		if ($plan !== null) {
+			[$pack, $offset, $length] = $this->index->get($id);
+			$encrypted = $plan->read('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
+			if ($encrypted !== null) {
+				try {
+					return $this->decodeBlob($id, $encrypted);
+				} catch (\Exception) {
+					// read again from the location below, which reports what is wrong
+				}
+			}
+		}
+		return $this->loadBlob($id);
+	}
+
 	private function loadBlob(string $id): string {
 		[$pack, $offset, $length] = $this->index->get($id);
 		try {