Backups to SMB no longer crawl; add occ backup:cancel
Each 30-second step opened the location again and read its list of index files. Over SMB that cost a request per file, so on a large repository opening took longer than the step: each step stored one 4 MiB pack and the list grew further. A first backup of nc01 to a Storage Box over SMB managed 12 GiB in 35 hours. occ backup:run and system cron now keep the repository open for the whole run (reopened when another process took a step or the location was edited), opening no longer counts towards the step's time, and SMB folders are listed in one request. occ backup:cancel stops a running backup: no new step starts and the step in progress finishes first. A run of an earlier version, which does not hold back its next step, is marked cancelled after two minutes unless it is recording its snapshot.
This commit is contained in:
@@ -223,12 +223,23 @@ final class StorageBackend implements IBackend {
|
||||
if (!$this->storage->is_dir($abs)) {
|
||||
return []; // not created yet
|
||||
}
|
||||
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
||||
$names = $this->smbNames($abs);
|
||||
if ($names !== null) {
|
||||
$result = [];
|
||||
foreach ($names as $name) {
|
||||
if (!self::isTemporary($name)) {
|
||||
$result[] = substr($abs . '/' . $name, $cut);
|
||||
}
|
||||
}
|
||||
sort($result);
|
||||
return $result;
|
||||
}
|
||||
$dh = $this->storage->opendir($abs);
|
||||
if ($dh === false) {
|
||||
// An existing folder read as empty would look like a new repository.
|
||||
throw new BackendException('Cannot list ' . $dir);
|
||||
}
|
||||
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
||||
$result = [];
|
||||
while (($name = readdir($dh)) !== false) {
|
||||
if ($name !== '.' && $name !== '..' && !self::isTemporary($name)) {
|
||||
@@ -240,6 +251,46 @@ final class StorageBackend implements IBackend {
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* The names in an SMB folder, read in one listing; null for other storages, with access
|
||||
* control lists on, or when the storage's internals are not as expected. The SMB storage's
|
||||
* opendir() asks the server for the attributes of each entry to leave out hidden ones: with
|
||||
* thousands of index files that took minutes. Hidden entries are still left out: names
|
||||
* starting with a dot, and, after asking, any name a repository does not use (letters,
|
||||
* digits, - and _), such as desktop.ini or Thumbs.db. The repository's own files have such
|
||||
* names: they are not checked one by one, which was the cost.
|
||||
*
|
||||
* @return list<string>|null
|
||||
*/
|
||||
private function smbNames(string $abs): ?array {
|
||||
if (!$this->storage instanceof \OCA\Files_External\Lib\Storage\SMB) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
/** @var list<string>|null */
|
||||
return (function (string $path): ?array {
|
||||
/** @psalm-suppress UndefinedThisPropertyFetch */
|
||||
if ($this->checkAcl) {
|
||||
return null;
|
||||
}
|
||||
$names = [];
|
||||
/** @psalm-suppress UndefinedThisPropertyFetch, UndefinedMethod */
|
||||
foreach ($this->share->dir(ltrim($this->buildPath($path), '/')) as $info) {
|
||||
$name = (string)$info->getName();
|
||||
/** @psalm-suppress UndefinedThisPropertyFetch */
|
||||
if (!$this->showHidden && (str_starts_with($name, '.')
|
||||
|| (preg_match('/^[A-Za-z0-9_-]+$/', $name) !== 1 && $info->isHidden()))) {
|
||||
continue;
|
||||
}
|
||||
$names[] = $name;
|
||||
}
|
||||
return $names;
|
||||
})->call($this->storage, $abs);
|
||||
} catch (\Throwable) {
|
||||
return null; // another SMB storage version, or an error: the usual way reports it
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run $op; when it throws, or $failed says its result may come from a dead connection, and
|
||||
* the SFTP connection turns out to be dead, run it once more on a new connection.
|
||||
|
||||
Reference in New Issue
Block a user