Index: refresh the cache after a verify in one pass over the blobs, not one per pack

This commit is contained in:
2026-10-11 14:31:57 +02:00
parent 8e273ac259
commit df732f185a
2 changed files with 12 additions and 7 deletions
+4
View File
@@ -20,6 +20,10 @@ instead carried by the app description, the git tag, the release, and this file.
overwriting the result. The lease now lasts at least 15 minutes, a step writes the run only if
no other process wrote it since the step loaded it (in one database statement), and recording
a snapshot that is already recorded is no longer an error.
- `occ backup:verify` on a large repository failed with "SFTP connection failed while opening
snapshots/…": after downloading the index it refreshed the local index cache with a pass over
all blobs for every pack, which took 29 minutes for 1 TB (27,547 packs, 400,000 blobs). The
location dropped the idle connection in the meantime. It is one pass now (under a second).
## [0.9.4-beta.5] - 2026-10-09
+8 -7
View File
@@ -110,14 +110,15 @@ final class BlobIndex {
}
if ($verify && $this->cache !== null) {
// Refresh the cache from what was just verified.
// One pass over the blobs: a pass per pack took half an hour on a 1 TB repository
// (27,547 packs x 400,000 blobs), long enough for the location to drop the idle
// connection before the check went on.
$byPack = [];
foreach ($this->blobs as $id => [$pack, $offset, $length, $raw, $flags]) {
$byPack[$pack][] = ['id' => $id, 'offset' => $offset, 'length' => $length, 'raw' => $raw, 'flags' => $flags];
}
$this->cache->remove($this->repositoryId, array_keys($this->cache->all($this->repositoryId)));
foreach ($this->packs() as $packId => $_) {
$entries = [];
foreach ($this->blobs as $id => [$pack, $offset, $length, $raw, $flags]) {
if ($pack === $packId) {
$entries[] = ['id' => $id, 'offset' => $offset, 'length' => $length, 'raw' => $raw, 'flags' => $flags];
}
}
foreach ($byPack as $packId => $entries) {
$this->cache->put($this->repositoryId, $packId, $entries);
}
return;