$roots one directory, or name => directory (walked in order) * @param list $exclude logical path prefixes to skip * @param array $meta extra fields for the snapshot meta record */ public static function start(string|array $roots, ?string $parent = null, string $label = '', array $exclude = [], array $meta = []): array { $roots = is_string($roots) ? ['' => rtrim($roots, '/')] : $roots; uksort($roots, fn ($a, $b) => TreeWalker::compare((string)$a, (string)$b)); // walk order = compare order return [ 'snapshot' => bin2hex(random_bytes(16)), 'roots' => $roots, 'exclude' => $exclude, 'parent' => $parent, 'label' => $label, 'meta' => $meta, 'started' => gmdate('c'), 'phase' => 'files', 'after' => null, 'parentPos' => 0, 'cur' => null, 'segments' => [], 'steps' => 0, 'stats' => ['files' => 0, 'bytes' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0], ]; } /** Run one step until $deadline (microtime(true)); returns the new state. */ public static function step(Repository $repo, array $state, float $deadline): array { $state['steps']++; if ($state['phase'] === 'files') { $stats = $state['stats']; $walker = new TreeWalker($state['roots'], $state['exclude'], function (string $logical) use (&$state, &$stats): void { self::recordUnreadable($state, $stats, $logical); }, true); $parent = new ParentCursor($repo, $state['parent'], $state['parentPos']); $lines = []; $emptyDirs = []; $folders = []; $finishedWalk = true; foreach ($walker->walk($state['after']) as $logical => $path) { $safe = mb_check_encoding($logical, 'UTF-8') && Repository::isSafePath($logical); if ($walker->atFolderWithContent()) { // Only its time: what is in it is reported by itself, and a bad name in the path of // each of those is what gets reported. The cursor stays for such a name, see below. $mtime = $safe ? @filemtime($path) : false; if ($safe && $mtime !== false) { $folders[] = json_encode([$logical, $mtime], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); $state['after'] = $logical; if (microtime(true) >= $deadline) { $finishedWalk = false; break; } } continue; } if (!$safe) { // A name a snapshot cannot hold (a backslash, a control character, not UTF-8). One // such path must not make a backup fail, or the whole snapshot unreadable later. // The cursor stays where it is: the state is JSON, which cannot hold such a name. A // next step meets it again, and it is reported once. self::recordBadName($state, $stats, $logical); continue; } if (is_dir($path)) { $mtime = @filemtime($path); if ($mtime === false) { continue; // removed while walking } $emptyDirs[] = json_encode([$logical, $mtime], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); $stats['emptyDirs'] = ($stats['emptyDirs'] ?? 0) + 1; $state['after'] = $logical; if (microtime(true) >= $deadline) { $finishedWalk = false; break; } continue; } $version = Repository::fileVersion($path); if ($version === 'missing') { continue; // deleted while walking } [$size, $mtime] = array_map('intval', explode(':', $version, 3)); $prev = $parent->find($logical); if ($state['cur'] === null && $prev !== null && $prev['s'] === $size && $prev['m'] === $mtime) { $blobs = $prev['b']; $stats['reused']++; } else { $cur = $state['cur']; if ($cur !== null && $cur['path'] === $logical && $cur['version'] !== $version) { // Changed since the previous step: start over, unless it keeps changing. if (self::countRetry($state, $stats, $logical)) { $cur['version'] = $version; } else { $cur = null; } } if ($cur === null || $cur['path'] !== $logical) { $cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version]; } $fh = @fopen($path, 'rb'); if ($fh === false) { self::recordUnreadable($state, $stats, $logical); $state['cur'] = null; continue; } fseek($fh, $cur['offset']); $complete = true; while (($data = self::readFull($fh, Repository::BLOB_SIZE)) !== '') { $cur['blobs'][] = $repo->storeData($data, $stats); $cur['offset'] += strlen($data); if (microtime(true) >= $deadline && !feof($fh)) { $complete = false; break; } } fclose($fh); $now = Repository::fileVersion($path); if ($now !== $cur['version']) { if (!self::countRetry($state, $stats, $logical)) { // Changed while reading: read it again from the start (now or next step). $state['cur'] = null; $finishedWalk = false; break; } // Keeps changing (a live database, VM image or log): keep what was read, // like any copy of a file in use, and report it instead of retrying forever. $cur['version'] = $now; } if (!$complete) { $state['cur'] = $cur; $finishedWalk = false; break; } $blobs = $cur['blobs']; $size = $cur['offset']; $state['cur'] = null; $stats['read']++; } $lines[] = json_encode(['p' => $logical, 's' => $size, 'm' => $mtime, 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR); $stats['files']++; $stats['bytes'] += $size; $state['after'] = $logical; if (microtime(true) >= $deadline) { $finishedWalk = false; break; } } if ($lines !== []) { $writer = $repo->blobWriter($stats); $writer->write(implode("\n", $lines) . "\n"); $state['segments'][] = $writer->finish(); } if ($emptyDirs !== []) { $writer = $repo->blobWriter($stats); $writer->write(implode("\n", $emptyDirs) . "\n"); $state['dirSegments'][] = $writer->finish(); } if ($folders !== []) { $writer = $repo->blobWriter($stats); $writer->write(implode("\n", $folders) . "\n"); $state['folderSegments'][] = $writer->finish(); } $repo->flushPacks(); $state['parentPos'] = $parent->position(); $state['stats'] = $stats; if ($finishedWalk) { // The snapshot is written in a step of its own, with its own time: after a long // run it reads back thousands of segments, and the run reports that it got here. $state['phase'] = 'finish'; } return $state; } if ($state['phase'] === 'finish') { // Every segment ends with a newline, so they read as one stream: fetched together, // several at a time where the location allows it. // Empty folders go in the meta record, not between the files: a version that does not // know them reads the snapshot as before, and only restores no empty folders. $emptyDirs = []; foreach (isset($state['dirSegments']) ? $repo->readLines(array_values(array_merge(...$state['dirSegments']))) : [] as $line) { if ($line !== '') { $emptyDirs[] = json_decode($line, true, 8, JSON_THROW_ON_ERROR); } } // The times of the other folders too, packed: every look at a snapshot's meta record reads // this, and a string of a few hundred KiB costs that far less than tens of thousands of arrays. $folders = isset($state['folderSegments']) ? implode("\n", iterator_to_array($repo->readLines(array_values(array_merge(...$state['folderSegments']))), false)) : ''; $repo->putSnapshot($state['snapshot'], ['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'], 'roots' => array_keys($state['roots']), 'stats' => $state['stats'], 'warnings' => self::warnings($state)] + ($emptyDirs === [] ? [] : ['emptyDirs' => $emptyDirs]) + ($folders === '' ? [] : ['folderTimes' => base64_encode((string)gzdeflate($folders))]) + $state['meta'], $repo->readLines(array_values(array_merge(...$state['segments'])))); $state['phase'] = 'commit'; } return $state; } /** * Progress of the files phase as a fraction (at most 0.99 until it is done), or null without an * estimate. Bytes of a large file that is still being read count too. * * @return array{fraction: float, approximate: bool}|null */ public static function progress(array $state): ?array { $estimate = $state['estimate'] ?? null; if (($state['phase'] ?? null) !== 'files' || !is_array($estimate) || ($estimate['bytes'] ?? 0) <= 0) { return null; } $done = (int)($state['stats']['bytes'] ?? 0) + (int)($state['cur']['offset'] ?? 0); return ['fraction' => min(0.99, $done / (int)$estimate['bytes']), 'approximate' => (bool)($estimate['approximate'] ?? true)]; } /** * progress() of a run in its files phase; in its database phase, the tables dumped so far. * * @return array{fraction: float, approximate: bool, tables?: int, totalTables?: int}|null */ public static function progressOf(\OCA\NgBackup\Db\Run $run): ?array { if ($run->getStatus() !== \OCA\NgBackup\Db\Run::RUNNING || !in_array($run->getPhase(), ['db', 'files'], true)) { return null; } try { $state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR); } catch (\JsonException) { return null; } if ($run->getPhase() === 'db') { // Tables dumped so far, as of the last step (none known before the first step ends) $tables = count($state['dump']['tables'] ?? []); return $tables > 0 ? ['fraction' => min(1.0, ($state['dump']['i'] ?? 0) / $tables), 'approximate' => false, 'tables' => (int)($state['dump']['i'] ?? 0), 'totalTables' => $tables] : null; } return is_array($state['files'] ?? null) ? self::progress($state['files']) : null; } /** * A file or directory that could not be read: it is missing from this snapshot. Remembered (first * 1000, deduplicated across steps) so the run can report it instead of silently succeeding. */ private static function recordUnreadable(array &$state, array &$stats, string $logical): void { if (isset($state['unreadable'][$logical])) { return; } if (count($state['unreadable'] ?? []) < 1000) { $state['unreadable'][$logical] = true; } $stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1; } /** * A file or directory left out because of its name. Remembered like an unreadable one, in a form * that fits the JSON state and is safe to show: control characters and invalid bytes replaced. */ private static function recordBadName(array &$state, array &$stats, string $logical): void { $shown = (string)preg_replace('/[\x00-\x1f\x7f]/', '?', mb_scrub($logical, 'UTF-8')); if (isset($state['badNames'][$shown])) { return; } if (count($state['badNames'] ?? []) < 1000) { $state['badNames'][$shown] = true; } $stats['badNames'] = ($stats['badNames'] ?? 0) + 1; } /** @return array{unreadable: list, badNames: list, changing: list} examples for the report (first 50 each) */ public static function warnings(array $state): array { return [ 'unreadable' => array_slice(array_map('strval', array_keys($state['unreadable'] ?? [])), 0, 50), 'badNames' => array_slice(array_map('strval', array_keys($state['badNames'] ?? [])), 0, 50), 'changing' => array_slice(array_map('strval', array_values($state['changing'] ?? [])), 0, 50), ]; } /** Retries of one file that changes while it is read, before its last read is kept anyway. */ public const MAX_RETRIES = 3; /** * Count a retry of $logical. Returns true once it has changed MAX_RETRIES times: the caller then * keeps the data it read, and the file is listed in $state['changing'] (first 50) for the report. */ private static function countRetry(array &$state, array &$stats, string $logical): bool { $stats['retried'] = ($stats['retried'] ?? 0) + 1; $n = (($state['retryPath'] ?? null) === $logical ? (int)($state['retryCount'] ?? 0) : 0) + 1; $state['retryPath'] = $logical; $state['retryCount'] = $n; if ($n < self::MAX_RETRIES) { return false; } if ($n === self::MAX_RETRIES) { $stats['changing'] = ($stats['changing'] ?? 0) + 1; if (count($state['changing'] ?? []) < 50) { $state['changing'][] = $logical; } } return true; } private static function readFull($fh, int $length): string { $buffer = ''; while (strlen($buffer) < $length && !feof($fh)) { $buffer .= (string)fread($fh, $length - strlen($buffer)); } return $buffer; } }