Phase 2: verify per-user exports, refuse SMB without the native extension, matrix user CI
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / psalm (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled
CI / integration-targets (push) Has been cancelled
CI / integration-user (32) (push) Has been cancelled
CI / integration-user (33) (push) Has been cancelled
CI / integration-user (34) (push) Has been cancelled
CI / integration-user (35) (push) Has been cancelled
CI / integration-full-restore (push) Has been cancelled
CI / unit (8.2) (push) Has been cancelled
CI / unit (8.3) (push) Has been cancelled
CI / unit (8.4) (push) Has been cancelled
CI / psalm (push) Has been cancelled
CI / integration (mariadb, 32) (push) Has been cancelled
CI / integration (mariadb, 33) (push) Has been cancelled
CI / integration (mariadb, 34) (push) Has been cancelled
CI / integration (mariadb, 35) (push) Has been cancelled
CI / integration (pgsql, 32) (push) Has been cancelled
CI / integration (pgsql, 33) (push) Has been cancelled
CI / integration (pgsql, 34) (push) Has been cancelled
CI / integration (pgsql, 35) (push) Has been cancelled
CI / integration (sqlite, 32) (push) Has been cancelled
CI / integration (sqlite, 33) (push) Has been cancelled
CI / integration (sqlite, 34) (push) Has been cancelled
CI / integration (sqlite, 35) (push) Has been cancelled
CI / integration-targets (push) Has been cancelled
CI / integration-user (32) (push) Has been cancelled
CI / integration-user (33) (push) Has been cancelled
CI / integration-user (34) (push) Has been cancelled
CI / integration-user (35) (push) Has been cancelled
CI / integration-full-restore (push) Has been cancelled
backup:verify accepts a user export path (users/<uid>/<export>): every blob its manifest references is checked the same way as for a snapshot, shallow or --deep. The SMB backend is refused at backup:target:add when the native smbclient PECL extension is not loaded, instead of registering and then failing the first backup with "Seek failed": with only the smbclient binary files_external falls back to a shell stream that cannot seek. The per-user integration job now runs on Nextcloud 32-35, not just 35: it calls user_migration's internal UserMigrationService, whose API may change between versions.
This commit is contained in:
@@ -63,19 +63,24 @@ jobs:
|
||||
|
||||
integration-user:
|
||||
# Per-user backup/restore via user_migration (occ backup:backup:user / backup:restore:user),
|
||||
# an optional dependency installed from the app store. Does not depend on the Nextcloud
|
||||
# version or database beyond what the main integration job already covers, so this runs once.
|
||||
# an optional dependency installed from the app store. It calls user_migration's internal
|
||||
# UserMigrationService, whose API can change between Nextcloud versions, so this runs on every
|
||||
# supported version (not the database, which the main integration job already covers).
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
nextcloud: ['32', '33', '34', '35']
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up a fresh Nextcloud instance
|
||||
run: |
|
||||
docker run -d --name ngbu -e SQLITE_DATABASE=nextcloud -e NEXTCLOUD_ADMIN_USER=admin \
|
||||
-e NEXTCLOUD_ADMIN_PASSWORD=admin-user-roundtrip -e NEXTCLOUD_TRUSTED_DOMAINS=localhost \
|
||||
-v "$PWD:/var/www/html/custom_apps/ng_backup:ro" nextcloud:35-apache
|
||||
-v "$PWD:/var/www/html/custom_apps/ng_backup:ro" nextcloud:${{ matrix.nextcloud }}-apache
|
||||
for i in $(seq 1 90); do docker exec -u www-data ngbu php occ status 2>/dev/null | grep -q "installed: true" && break; sleep 4; done
|
||||
docker exec ngbu chown www-data:www-data /var/www/html/custom_apps
|
||||
- name: Per-user backup/restore round trip
|
||||
- name: Per-user backup/restore round trip on Nextcloud ${{ matrix.nextcloud }}
|
||||
run: tests/integration/user-roundtrip.sh "docker exec -u www-data ngbu" /var/www/html/data
|
||||
|
||||
integration-full-restore:
|
||||
|
||||
@@ -49,13 +49,17 @@ HELP);
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
try {
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$result = $this->verify->verify($target, (string)$input->getArgument('snapshot'), (bool)$input->getOption('deep'));
|
||||
$ref = (string)$input->getArgument('snapshot');
|
||||
$deep = (bool)$input->getOption('deep');
|
||||
$result = str_starts_with($ref, 'users/')
|
||||
? $this->verify->verifyUserExport($target, $ref, $deep)
|
||||
: $this->verify->verify($target, $ref, $deep);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln(sprintf('%d files, %d blobs checked%s.', $result['filesChecked'], $result['blobsChecked'],
|
||||
$input->getOption('deep') ? sprintf(' (%.1f MiB decrypted)', $result['bytesChecked'] / 1048576) : ''));
|
||||
$deep ? sprintf(' (%.1f MiB decrypted)', $result['bytesChecked'] / 1048576) : ''));
|
||||
if ($result['missing'] !== []) {
|
||||
$output->writeln('<error>' . count($result['missing']) . ' blob(s) missing from the index:</error> ' . implode(', ', $result['missing']));
|
||||
}
|
||||
|
||||
@@ -716,6 +716,40 @@ final class Repository {
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Same checks as verify(), for a user export: every blob its manifest references must be in
|
||||
* the freshly re-authenticated index with its pack present; $deep also decrypts each.
|
||||
*
|
||||
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>}
|
||||
*/
|
||||
public function verifyUserExport(string $manifestPath, bool $deep = false, ?callable $heartbeat = null): array {
|
||||
if (!$this->isRecordedUserExport($manifestPath)) {
|
||||
throw new RepositoryException("$manifestPath is not a recorded user export for this location");
|
||||
}
|
||||
$heartbeat ??= static function (): void {
|
||||
};
|
||||
$this->index->load(true);
|
||||
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => []];
|
||||
$seen = [];
|
||||
$packSeen = [];
|
||||
$manifest = json_decode($this->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
||||
foreach ($manifest['entries'] as $entry) {
|
||||
if (($entry['t'] ?? '') !== 'f') {
|
||||
continue;
|
||||
}
|
||||
$stats['filesChecked']++;
|
||||
foreach ($entry['b'] ?? [] as $id) {
|
||||
if (isset($seen[$id])) {
|
||||
continue;
|
||||
}
|
||||
$seen[$id] = true;
|
||||
$this->verifyBlob($id, $deep, $stats, $packSeen);
|
||||
$heartbeat();
|
||||
}
|
||||
}
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/** @param array<string, bool> $packSeen */
|
||||
private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen): void {
|
||||
if (!$this->index->has($id)) {
|
||||
|
||||
@@ -53,6 +53,12 @@ final class TargetService {
|
||||
if (!preg_match('/^[A-Za-z0-9][A-Za-z0-9 _.-]{0,63}$/', $name)) {
|
||||
throw new \InvalidArgumentException('Name: letters, digits, space, _ . - (max 64)');
|
||||
}
|
||||
// Without the native extension files_external falls back to the smbclient binary, which
|
||||
// registers fine but cannot seek: the first backup then fails with "Seek failed". Refuse
|
||||
// up front instead of letting the admin find out from a failed run.
|
||||
if ($backend === 'smb' && !extension_loaded('smbclient')) {
|
||||
throw new \InvalidArgumentException('The SMB backend needs the PHP smbclient extension (pecl install smbclient); the smbclient binary alone cannot seek and every backup would fail');
|
||||
}
|
||||
try {
|
||||
$this->mapper->findByName($name);
|
||||
throw new \InvalidArgumentException("A target named '$name' already exists");
|
||||
|
||||
@@ -20,9 +20,18 @@ final class VerifyService {
|
||||
|
||||
/** @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>} */
|
||||
public function verify(Target $target, string $snapshotId, bool $deep = false): array {
|
||||
return $this->run($target, $deep, fn (\OCA\NgBackup\Repository\Repository $repo, callable $heartbeat): array => $repo->verify($snapshotId, $deep, $heartbeat));
|
||||
}
|
||||
|
||||
/** @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>} */
|
||||
public function verifyUserExport(Target $target, string $manifestPath, bool $deep = false): array {
|
||||
return $this->run($target, $deep, fn (\OCA\NgBackup\Repository\Repository $repo, callable $heartbeat): array => $repo->verifyUserExport($manifestPath, $deep, $heartbeat));
|
||||
}
|
||||
|
||||
private function run(Target $target, bool $deep, callable $fn): array {
|
||||
$ttl = $deep ? 600 : 300;
|
||||
try {
|
||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, function (callable $refresh) use ($target, $snapshotId, $deep, $ttl) {
|
||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, function (callable $refresh) use ($target, $deep, $ttl, $fn): array {
|
||||
$last = time();
|
||||
$heartbeat = function () use ($refresh, $ttl, &$last): void {
|
||||
if (time() - $last >= intdiv($ttl, 2)) {
|
||||
@@ -30,7 +39,7 @@ final class VerifyService {
|
||||
$last = time();
|
||||
}
|
||||
};
|
||||
return $this->targets->repository($target)->verify($snapshotId, $deep, $heartbeat);
|
||||
return $fn($this->targets->repository($target), $heartbeat);
|
||||
});
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||
|
||||
Reference in New Issue
Block a user