Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Verify: index files several at a time over SFTP, pack folders listed, progress stored and shown

Browse Source
This commit is contained in:
Dillard Blom
2026-10-11 13:39:38 +02:00
parent 828f3f142a
commit 379b81db02
15 changed files with 600 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Repository/BlobIndex.php
+28 -11
View File
@@ -87,27 +87,37 @@ final class BlobIndex {
* and only the index files that are not cached yet are downloaded. Cached packs that no
* longer exist on the backend are dropped from the cache.
*/
public function load(bool $verify = false): void {
public function load(bool $verify = false, ?callable $progress = null): void {
$this->blobs = [];
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
$present = [];
$this->downloaded = 0;
$fetch = [];
$entriesByPack = []; // in listing order: where a blob is in two packs, the last one listed wins
foreach ($this->backend->listFiles('index') as $path) {
$packId = basename($path);
$present[$packId] = true;
if (isset($cached[$packId])) {
$entries = $cached[$packId];
} else {
$fh = $this->backend->get($path);
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
fclose($fh);
$this->downloaded++;
$this->cache?->put($this->repositoryId, $packId, $entries);
$entriesByPack[$packId] = $cached[$packId] ?? null;
if (!isset($cached[$packId])) {
$fetch[] = $path;
}
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
}
// The rest is downloaded several at a time where the location allows it: on a large
// repository these are tens of thousands of small files.
$total = count($fetch);
foreach ($this->backend->getMany($fetch) as $path => $data) {
$packId = basename($path);
$entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
$this->downloaded++;
$this->cache?->put($this->repositoryId, $packId, $entries);
$entriesByPack[$packId] = $entries;
if ($progress !== null) {
$progress($this->downloaded, $total);
}
}
foreach ($entriesByPack as $packId => $entries) {
$this->add($packId, $entries ?? []);
}
if ($verify && $this->cache !== null) {
// Refresh the cache from what was just verified.
$this->cache->remove($this->repositoryId, array_keys($this->cache->all($this->repositoryId)));
@@ -128,6 +138,13 @@ final class BlobIndex {
}
}
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
private function add(string $packId, array $entries): void {
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
}
}
public function has(string $id): bool {
return isset($this->blobs[$id]) || isset($this->pending[$id]);
}