Index: refresh the cache after a verify in one pass over the blobs, not one per pack

This commit is contained in:
2026-10-11 14:31:57 +02:00
parent 8e273ac259
commit df732f185a
2 changed files with 12 additions and 7 deletions
+4
View File
@@ -20,6 +20,10 @@ instead carried by the app description, the git tag, the release, and this file.
overwriting the result. The lease now lasts at least 15 minutes, a step writes the run only if overwriting the result. The lease now lasts at least 15 minutes, a step writes the run only if
no other process wrote it since the step loaded it (in one database statement), and recording no other process wrote it since the step loaded it (in one database statement), and recording
a snapshot that is already recorded is no longer an error. a snapshot that is already recorded is no longer an error.
- `occ backup:verify` on a large repository failed with "SFTP connection failed while opening
snapshots/…": after downloading the index it refreshed the local index cache with a pass over
all blobs for every pack, which took 29 minutes for 1 TB (27,547 packs, 400,000 blobs). The
location dropped the idle connection in the meantime. It is one pass now (under a second).
## [0.9.4-beta.5] - 2026-10-09 ## [0.9.4-beta.5] - 2026-10-09
+8 -7
View File
@@ -110,14 +110,15 @@ final class BlobIndex {
} }
if ($verify && $this->cache !== null) { if ($verify && $this->cache !== null) {
// Refresh the cache from what was just verified. // Refresh the cache from what was just verified.
// One pass over the blobs: a pass per pack took half an hour on a 1 TB repository
// (27,547 packs x 400,000 blobs), long enough for the location to drop the idle
// connection before the check went on.
$byPack = [];
foreach ($this->blobs as $id => [$pack, $offset, $length, $raw, $flags]) {
$byPack[$pack][] = ['id' => $id, 'offset' => $offset, 'length' => $length, 'raw' => $raw, 'flags' => $flags];
}
$this->cache->remove($this->repositoryId, array_keys($this->cache->all($this->repositoryId))); $this->cache->remove($this->repositoryId, array_keys($this->cache->all($this->repositoryId)));
foreach ($this->packs() as $packId => $_) { foreach ($byPack as $packId => $entries) {
$entries = [];
foreach ($this->blobs as $id => [$pack, $offset, $length, $raw, $flags]) {
if ($pack === $packId) {
$entries[] = ['id' => $id, 'offset' => $offset, 'length' => $length, 'raw' => $raw, 'flags' => $flags];
}
}
$this->cache->put($this->repositoryId, $packId, $entries); $this->cache->put($this->repositoryId, $packId, $entries);
} }
return; return;