Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Verify: index files several at a time over SFTP, pack folders listed, progress stored and shown

Browse Source
This commit is contained in:
Dillard Blom
2026-10-11 13:39:38 +02:00
parent 828f3f142a
commit 379b81db02
15 changed files with 600 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Repository/BlobIndex.php
+28 -11
View File
@@ -87,27 +87,37 @@ final class BlobIndex {
* and only the index files that are not cached yet are downloaded. Cached packs that no
* longer exist on the backend are dropped from the cache.
*/
public function load(bool $verify = false): void {
public function load(bool $verify = false, ?callable $progress = null): void {
$this->blobs = [];
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
$present = [];
$this->downloaded = 0;
$fetch = [];
$entriesByPack = []; // in listing order: where a blob is in two packs, the last one listed wins
foreach ($this->backend->listFiles('index') as $path) {
$packId = basename($path);
$present[$packId] = true;
if (isset($cached[$packId])) {
$entries = $cached[$packId];
} else {
$fh = $this->backend->get($path);
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
fclose($fh);
$this->downloaded++;
$this->cache?->put($this->repositoryId, $packId, $entries);
$entriesByPack[$packId] = $cached[$packId] ?? null;
if (!isset($cached[$packId])) {
$fetch[] = $path;
}
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
}
// The rest is downloaded several at a time where the location allows it: on a large
// repository these are tens of thousands of small files.
$total = count($fetch);
foreach ($this->backend->getMany($fetch) as $path => $data) {
$packId = basename($path);
$entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
$this->downloaded++;
$this->cache?->put($this->repositoryId, $packId, $entries);
$entriesByPack[$packId] = $entries;
if ($progress !== null) {
$progress($this->downloaded, $total);
}
}
foreach ($entriesByPack as $packId => $entries) {
$this->add($packId, $entries ?? []);
}
if ($verify && $this->cache !== null) {
// Refresh the cache from what was just verified.
$this->cache->remove($this->repositoryId, array_keys($this->cache->all($this->repositoryId)));
@@ -128,6 +138,13 @@ final class BlobIndex {
}
}
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
private function add(string $packId, array $entries): void {
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
}
}
public function has(string $id): bool {
return isset($this->blobs[$id]) || isset($this->pending[$id]);
}
lib/Repository/PackReaderGuard.php
+5
View File
@@ -28,6 +28,11 @@ final class PackReaderGuard implements IBackend {
return $this->inner->get($path);
}
public function getMany(array $paths): \Generator {
$this->reader->close();
return $this->inner->getMany($paths);
}
public function getRange(string $path, int $offset, int $length): string {
$this->reader->close();
return $this->inner->getRange($path, $offset, $length);
lib/Repository/Repository.php
+57 -13
View File
@@ -754,13 +754,22 @@ final class Repository {
* callers use it to keep a time-boxed lease alive during a large audit
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
*/
/**
* @param ?callable(array{phase:string, done:int, total:?int, blobs?:int, bytes?:int}):void $heartbeat
* called often during the check, with how far it is: phase "index" (index files
* downloaded), "packs" (pack folders listed), "files" (files of the snapshot checked)
*/
public function verify(string $snapshotId, bool $deep = false, ?callable $heartbeat = null): array {
$heartbeat ??= static function (): void {
$heartbeat ??= static function (array $progress): void {
};
$this->index->load(true);
$meta = $this->snapshotMeta($snapshotId);
$filesTotal = isset($meta['stats']['files']) ? (int)$meta['stats']['files'] : null;
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
});
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
$seen = [];
$packSeen = []; // pack id => exists on the backend; avoids one exists() call per blob
[$packSeen, $listedDirs] = $this->presentPacks($heartbeat);
foreach ($this->streamEntries($snapshotId) as $entry) {
$stats['filesChecked']++;
foreach ($entry['b'] as $id) {
@@ -768,11 +777,11 @@ final class Repository {
continue;
}
$seen[$id] = true;
$this->verifyBlob($id, $deep, $stats, $packSeen);
$heartbeat();
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
}
}
$meta = $this->snapshotMeta($snapshotId);
if (isset($meta['db']) && is_string($meta['db'])) {
if (!$this->backend->exists($meta['db'])) {
$stats['missing'][] = $meta['db'];
@@ -784,8 +793,9 @@ final class Repository {
continue;
}
$seen[$id] = true;
$this->verifyBlob($id, $deep, $stats, $packSeen);
$heartbeat();
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
$heartbeat(['phase' => 'database', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
}
}
}
@@ -793,6 +803,30 @@ final class Repository {
return $stats;
}
/**
* Which packs are on the location, from one listing per pack folder instead of a request per
* pack. A folder that cannot be listed is left out: its packs are then checked one by one.
*
* @return array{0: array<string, bool>, 1: array<string, true>} pack id => present, and the
* folders that were listed
*/
private function presentPacks(callable $heartbeat): array {
$present = [];
$listed = [];
for ($i = 0; $i < 256; $i++) {
$dir = sprintf('%02x', $i);
try {
foreach ($this->backend->listFiles('packs/' . $dir) as $path) {
$present[basename($path)] = true;
}
$listed[$dir] = true;
} catch (BackendException) {
}
$heartbeat(['phase' => 'packs', 'done' => $i + 1, 'total' => 256]);
}
return [$present, $listed];
}
/**
* Same checks as verify(), for a user export: every blob its manifest references must be in
* the freshly re-authenticated index with its pack present; $deep also decrypts each.
@@ -803,9 +837,11 @@ final class Repository {
if (!$this->isRecordedUserExport($manifestPath)) {
throw new RepositoryException("$manifestPath is not a recorded user export for this location");
}
$heartbeat ??= static function (): void {
$heartbeat ??= static function (array $progress): void {
};
$this->index->load(true);
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
});
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
$seen = [];
$packSeen = [];
@@ -821,20 +857,28 @@ final class Repository {
}
$seen[$id] = true;
$this->verifyBlob($id, $deep, $stats, $packSeen);
$heartbeat();
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => null,
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
}
}
return $stats;
}
/** @param array<string, bool> $packSeen */
private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen): void {
/**
* @param array<string, bool> $packSeen pack id => present; filled in as packs are checked
* @param array<string, true> $listedDirs pack folders whose listing is in $packSeen: a pack
* not in it is not there
*/
private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen, array $listedDirs = []): void {
if (!$this->index->has($id)) {
$stats['missing'][] = $id;
return;
}
[$pack] = $this->index->get($id);
$packExists = $packSeen[$pack] ??= $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
$packExists = $packSeen[$pack] ??= isset($listedDirs[substr($pack, 0, 2)])
? false
: $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
if (!$packExists) {
$stats['missing'][] = $id;
return;