Index: refresh the cache after a verify in one pass over the blobs, not one per pack
This commit is contained in:
@@ -20,6 +20,10 @@ instead carried by the app description, the git tag, the release, and this file.
|
||||
overwriting the result. The lease now lasts at least 15 minutes, a step writes the run only if
|
||||
no other process wrote it since the step loaded it (in one database statement), and recording
|
||||
a snapshot that is already recorded is no longer an error.
|
||||
- `occ backup:verify` on a large repository failed with "SFTP connection failed while opening
|
||||
snapshots/…": after downloading the index it refreshed the local index cache with a pass over
|
||||
all blobs for every pack, which took 29 minutes for 1 TB (27,547 packs, 400,000 blobs). The
|
||||
location dropped the idle connection in the meantime. It is one pass now (under a second).
|
||||
|
||||
## [0.9.4-beta.5] - 2026-10-09
|
||||
|
||||
|
||||
@@ -110,14 +110,15 @@ final class BlobIndex {
|
||||
}
|
||||
if ($verify && $this->cache !== null) {
|
||||
// Refresh the cache from what was just verified.
|
||||
// One pass over the blobs: a pass per pack took half an hour on a 1 TB repository
|
||||
// (27,547 packs x 400,000 blobs), long enough for the location to drop the idle
|
||||
// connection before the check went on.
|
||||
$byPack = [];
|
||||
foreach ($this->blobs as $id => [$pack, $offset, $length, $raw, $flags]) {
|
||||
$byPack[$pack][] = ['id' => $id, 'offset' => $offset, 'length' => $length, 'raw' => $raw, 'flags' => $flags];
|
||||
}
|
||||
$this->cache->remove($this->repositoryId, array_keys($this->cache->all($this->repositoryId)));
|
||||
foreach ($this->packs() as $packId => $_) {
|
||||
$entries = [];
|
||||
foreach ($this->blobs as $id => [$pack, $offset, $length, $raw, $flags]) {
|
||||
if ($pack === $packId) {
|
||||
$entries[] = ['id' => $id, 'offset' => $offset, 'length' => $length, 'raw' => $raw, 'flags' => $flags];
|
||||
}
|
||||
}
|
||||
foreach ($byPack as $packId => $entries) {
|
||||
$this->cache->put($this->repositoryId, $packId, $entries);
|
||||
}
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user