Restore: follow blobs that another server's cleanup repacked during the restore

CHANGELOG.md

@@ -39,6 +39,10 @@ All-in-One onto a plain Nextcloud with PostgreSQL).
 - `occ backup:target:add` without a key pointed to `backup:key:init`, which makes a new key that
   cannot read existing backups. It now points to `backup:key:import` for a restore, and says that
   connecting to a location with backups can take a while.
+- A restore stopped with "Not found: packs/..." when the location was cleaned up while it ran,
+  for example by the weekly cleanup of the original server during a recovery test on a second
+  server: blobs that were moved into new packs were still looked for in the old ones. A restore
+  now reloads the index and reads them from where they are now.
 - Removing a location left its cached index in the database (tens of thousands of rows for a large
   backup). It is removed with the location now, and once for locations that are already gone.
 

lib/Repository/Repository.php

@@ -771,16 +771,48 @@ final class Repository {
 			$ranges[] = [$curPath, $curOffset, $curLength];
 			$parts[] = $curParts;
 		}
-		foreach ($this->backend->getRanges($ranges) as $i => $data) {
+		$done = 0;
-			foreach ($parts[$i] as [$id, $at, $length]) {
+		try {
-				yield $this->decodeBlob($id, substr($data, $at, $length));
+			foreach ($this->backend->getRanges($ranges) as $i => $data) {
+				foreach ($parts[$i] as [$id, $at, $length]) {
+					yield $this->decodeBlob($id, substr($data, $at, $length));
+					$done++;
+				}
+			}
+		} catch (BackendException $e) {
+			// A pack moved (see loadBlob()): the rest one blob at a time, from the fresh index.
+			$this->refreshIndex($e);
+			foreach (array_slice($ids, $done) as $id) {
+				yield $this->loadBlob($id);
 			}
 		}
 	}
 
 	private function loadBlob(string $id): string {
 		[$pack, $offset, $length] = $this->index->get($id);
-		return $this->decodeBlob($id, $this->packReader->read($pack, $offset, $length));
+		try {
+			$encrypted = $this->packReader->read($pack, $offset, $length);
+		} catch (BackendException $e) {
+			// Another server's cleanup may have repacked this blob into a new pack since the index
+			// was loaded (a disaster recovery test while the original server still runs). Reload
+			// the index from the location; read again only if the blob has moved.
+			$this->refreshIndex($e);
+			[$moved, $offset, $length] = $this->index->get($id);
+			if ($moved === $pack) {
+				throw $e;
+			}
+			$encrypted = $this->packReader->read($moved, $offset, $length);
+		}
+		return $this->decodeBlob($id, $encrypted);
+	}
+
+	/** Reload the index from the location after a failed pack read; rethrows that failure if it cannot. */
+	private function refreshIndex(BackendException $cause): void {
+		try {
+			$this->index->load();
+		} catch (\Throwable) {
+			throw $cause;
+		}
 	}
 
 	/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */

tests/Unit/Repository/PruneTest.php

@@ -88,6 +88,33 @@ class PruneTest extends TestCase {
 		$this->assertSame(self::hashTree($src), self::hashTree($target));
 	}
 
+	public function testRestoreFollowsPacksRepackedByAnotherServer(): void {
+		// A disaster recovery test on a second server reads the location while the original
+		// server's weekly cleanup repacks it: the restore must not stop at a pack that moved.
+		$src = $this->tempDir();
+		$backend = new LocalBackend($this->tempDir());
+		$keys = KeyRing::generate();
+		$repo = Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
+		for ($i = 0; $i < 40; $i++) {
+			file_put_contents("$src/f$i", random_bytes(500000));
+		}
+		$a = $repo->backupDirectory($src);
+		for ($i = 0; $i < 40; $i++) {
+			if ($i % 5 !== 0) {
+				unlink("$src/f$i");
+			}
+		}
+		$b = $repo->backupDirectory($src, $a['snapshot']);
+		$reader = Repository::openWithKey($backend, $keys); // index loaded before the cleanup
+		$repo->forget($a['snapshot']);
+		$repo->purgeTrash(0);
+		$this->assertGreaterThan(0, $repo->prune(false, 0.5)['repacked']);
+
+		$target = $this->tempDir();
+		$reader->restore($b['snapshot'], $target);
+		$this->assertSame(self::hashTree($src), self::hashTree($target));
+	}
+
 	public function testUntrashRestoresAWorkingSnapshot(): void {
 		$src = $this->tempDir();
 		$backend = new LocalBackend($this->tempDir());