Over SMB, read index files in fewer round trips

files_external's fopen() first stats the file and icewind's stream reads 1 MiB at a time. Open,
read and close straight through the native extension's state instead: from Canada to a Hetzner
Storage Box (130 ms) 0.66 instead of 1.05 s per index file, about 5 instead of 8 hours for the
27,000 index files of a 1 TB repository in backup:target:add. Anything unexpected falls back to
the usual way.

CHANGELOG.md

@@ -34,6 +34,10 @@ shown under 1.0.0. The git tag of a release candidate goes after the date.
 - Full restores from S3 read the packs ahead, eight ranges at a time, as restores over SFTP do
   since 1.0.0-rc.4. Over SMB the packs are read in 8 MiB reads instead of 1 MiB: over a line with
   a 130 ms round trip, reading was four times faster in a test (18 instead of 4.5 MiB/s).
+- Over SMB the index files, which a new server reads in `backup:target:add`, take fewer round
+  trips: from Canada to a Hetzner Storage Box (130 ms),
+  0.66 instead of 1.05 s per file, so about 5 instead of 8 hours for the 27,000 index files of a
+  1 TB repository.
 - `occ backup:target:add` only estimates the time left for reading the index once enough of it
   is in. The first file also carries the time to connect, which made it say "about 642 min left"
   for something done within a minute.

lib/Backend/StorageBackend.php

@@ -133,7 +133,7 @@ final class StorageBackend implements IBackend, IReadsAhead {
 
 	/**
 	 * Over SFTP and S3 several at a time (SftpReadStream::readMany(), s3Fetch()); elsewhere one
-	 * after the other.
+	 * after the other, over SMB in fewer round trips (smbWhole()).
 	 */
 	public function getMany(array $paths): \Generator {
 		$size = $this->s3Reads ? 16 * self::S3_CONCURRENCY : 8 * SftpReadStream::BATCH;
@@ -149,7 +149,8 @@ final class StorageBackend implements IBackend, IReadsAhead {
 				$fast = $fast === null ? null : array_combine($chunk, array_map(fn (string $a): ?string => $fast[$a] ?? null, $abs));
 			}
 			foreach ($chunk as $path) {
-				$data = $fast[$path] ?? null;
+				// over SMB in fewer round trips, one after the other (smbWhole())
+				$data = $fast[$path] ?? $this->smbWhole($this->abs($path));
 				if ($data === null) {
 					// not read in the batch (or no batches here): the usual way, which also
 					// reports a missing file as such
@@ -543,25 +544,7 @@ final class StorageBackend implements IBackend, IReadsAhead {
 	 * @psalm-suppress UndefinedClass icewind/smb comes with files_external, not with this app
 	 */
 	private function smbChunks(string $abs, int $offset, int $length): ?\Generator {
-		if (!$this->storage instanceof \OCA\Files_External\Lib\Storage\SMB || !extension_loaded('smbclient')) {
-			return null;
-		}
-		try {
-			/** @var array{0: object, 1: string}|null $handle */
-			$handle = (function (string $path): ?array {
-				/** @psalm-suppress UndefinedThisPropertyFetch */
-				$share = $this->share;
-				if (!$share instanceof \Icewind\SMB\Native\NativeShare) {
-					return null;
-				}
-				/** @psalm-suppress UndefinedMethod */
-				$sharePath = ltrim($this->buildPath($path), '/');
-				/** @psalm-suppress UndefinedMethod */
-				return (fn (string $p): array => [$this->getState(), $this->buildUrl($p)])->call($share, $sharePath);
-			})->call($this->storage, $abs);
-		} catch (\Throwable) {
-			return null;
-		}
+		$handle = $this->smbHandle($abs);
 		if ($handle === null) {
 			return null;
 		}
@@ -585,6 +568,78 @@ final class StorageBackend implements IBackend, IReadsAhead {
 		})();
 	}
 
+	/**
+	 * A whole (small) file over SMB in four round trips: open, read, read up to the end, close.
+	 * files_external's fopen() first checks that the file exists (a stat with its attributes) and
+	 * icewind's stream reads 1 MiB at a time: about 7 round trips, 1 s per index file from Canada
+	 * to the Storage Box (130 ms), so ~8 h for the 27,000 index files of a 1 TB repository. Null
+	 * when the storage is not SMB through the extension or anything fails: the usual way then
+	 * reads the file and reports a missing one as such.
+	 *
+	 * @psalm-suppress UndefinedClass icewind/smb comes with files_external, not with this app
+	 */
+	private function smbWhole(string $abs): ?string {
+		$handle = $this->smbHandle($abs);
+		if ($handle === null) {
+			return null;
+		}
+		[$state, $url] = $handle;
+		try {
+			/** @psalm-suppress MixedMethodCall the state is icewind's NativeState */
+			$fh = $state->open($url, 'r');
+		} catch (\Throwable) {
+			return null;
+		}
+		try {
+			$data = '';
+			do {
+				/** @psalm-suppress MixedMethodCall */
+				$chunk = (string)$state->read($fh, self::SMB_READ, $url);
+				$data .= $chunk;
+			} while ($chunk !== '');
+			return $data;
+		} catch (\Throwable) {
+			return null;
+		} finally {
+			try {
+				/** @psalm-suppress MixedMethodCall */
+				$state->close($fh, $url);
+			} catch (\Throwable) {
+			}
+		}
+	}
+
+	/**
+	 * icewind's NativeState and the smb:// URL of $abs behind files_external's SMB storage, reached
+	 * through bound closures; null when the storage is not SMB through the native extension or its
+	 * internals are not as expected.
+	 *
+	 * @return array{0: object, 1: string}|null
+	 * @psalm-suppress UndefinedClass icewind/smb comes with files_external, not with this app
+	 */
+	private function smbHandle(string $abs): ?array {
+		if (!$this->storage instanceof \OCA\Files_External\Lib\Storage\SMB || !extension_loaded('smbclient')) {
+			return null;
+		}
+		try {
+			/** @var array{0: object, 1: string}|null $handle */
+			$handle = (function (string $path): ?array {
+				/** @psalm-suppress UndefinedThisPropertyFetch */
+				$share = $this->share;
+				if (!$share instanceof \Icewind\SMB\Native\NativeShare) {
+					return null;
+				}
+				/** @psalm-suppress UndefinedMethod */
+				$sharePath = ltrim($this->buildPath($path), '/');
+				/** @psalm-suppress UndefinedMethod */
+				return (fn (string $p): array => [$this->getState(), $this->buildUrl($p)])->call($share, $sharePath);
+			})->call($this->storage, $abs);
+		} catch (\Throwable) {
+			return null;
+		}
+		return $handle;
+	}
+
 	/**
 	 * readAhead() over SMB: one range after the other, each in SMB_READ reads into a file in the
 	 * spool. Not several at once (PHP cannot wait on more than one libsmbclient call), but whole