SFTP: replace a dropped connection; write the index cache in one transaction

A Hetzner Storage Box drops an SFTP connection that carried a lot of data
and then stayed quiet for a few minutes; files_external keeps using the dead
client, on which every file looks missing. StorageBackend now checks the
connection after a minute without requests, and replaces a connection that
is dead after a failed request (once per request; put, move and delete are
not repeated). SftpReadStream closes a connection it left in an unknown state.

The verify cache refresh took more than 6 minutes on PostgreSQL for 27,547
packs, one autocommitted insert per pack: putMany()/replaceAll() write in one
transaction. After a verify the cache holds the index files as they are, so a
blob in two packs stays cached for both.

lib/Repository/BlobIndex.php

@@ -28,6 +28,8 @@ final class BlobIndex {
 	/** @var list<\Closure():void>|null cache writes held back by holdCache(), or null when writing directly */
 	private ?array $heldCacheWrites = null;
 	private string $repositoryId = '';
+	/** Index files written to the cache per database transaction while loading. */
+	private const CACHE_LOT = 256;
 	/** @var int index files downloaded by the last load() (for tests and diagnostics) */
 	public int $downloaded = 0;
 
@@ -107,11 +109,19 @@ final class BlobIndex {
 		// the cached packs listed before each one are added first.
 		$next = 0;
 		$total = count($fetch);
+		$toCache = []; // written to the cache a few hundred at a time, each lot in one transaction
 		foreach ($this->backend->getMany($fetch) as $path => $data) {
 			$packId = basename($path);
 			$entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
 			$this->downloaded++;
-			$this->cache?->put($this->repositoryId, $packId, $entries);
+			if ($this->cache !== null) {
+				$toCache[$packId] = $entries;
+				// a check writes the whole cache afresh at the end
+				if (!$verify && count($toCache) >= self::CACHE_LOT) {
+					$this->cache->putMany($this->repositoryId, $toCache);
+					$toCache = [];
+				}
+			}
 			for (; $next < count($order) && $order[$next] !== $packId; $next++) {
 				$this->add($order[$next], $cached[$order[$next]] ?? []);
 			}
@@ -125,20 +135,16 @@ final class BlobIndex {
 			$this->add($order[$next], $cached[$order[$next]] ?? []);
 		}
 		if ($verify && $this->cache !== null) {
-			// Refresh the cache from what was just verified.
-			// One pass over the blobs: a pass per pack took half an hour on a 1 TB repository
-			// (27,547 packs x 400,000 blobs), long enough for the location to drop the idle
-			// connection before the check went on.
-			$byPack = [];
-			foreach ($this->blobs as $id => [$pack, $offset, $length, $raw, $flags]) {
-				$byPack[$pack][] = ['id' => $id, 'offset' => $offset, 'length' => $length, 'raw' => $raw, 'flags' => $flags];
-			}
-			$this->cache->remove($this->repositoryId, array_keys($this->cache->all($this->repositoryId)));
-			foreach ($byPack as $packId => $entries) {
-				$this->cache->put($this->repositoryId, $packId, $entries);
-			}
+			// Refresh the cache from what was just verified, as the index files list it: a blob in
+			// two packs is in both rows, so that removing one of the packs leaves the other right.
+			// In one transaction: one row at a time took more than 6 minutes on a 1 TB repository
+			// (27,547 packs), long enough for the location to drop the connection.
+			$this->cache->replaceAll($this->repositoryId, $toCache);
 			return;
 		}
+		if ($toCache !== []) {
+			$this->cache?->putMany($this->repositoryId, $toCache);
+		}
 		$stale = array_keys(array_diff_key($cached, $present));
 		if ($stale !== []) {
 			$this->cache?->remove($this->repositoryId, $stale);

lib/Repository/IIndexCache.php

@@ -15,6 +15,20 @@ interface IIndexCache {
 	/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
 	public function put(string $repositoryId, string $packId, array $entries): void;
 
+	/**
+	 * put() for many packs at once (one database transaction).
+	 *
+	 * @param array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> $packs packId => entries
+	 */
+	public function putMany(string $repositoryId, array $packs): void;
+
+	/**
+	 * Replace everything cached for the repository with $packs (one database transaction).
+	 *
+	 * @param array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> $packs packId => entries
+	 */
+	public function replaceAll(string $repositoryId, array $packs): void;
+
 	/** @param list<string> $packIds */
 	public function remove(string $repositoryId, array $packIds): void;
 }