Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Pre-beta review: data safety in storage, catalog and backup runs

Browse Source 
Storage and repository:
- Listing skipped every name containing ".part-", directories too, so the
  exports of a user like "john.part-time" were invisible to prune on S3,
  SFTP, FTP and SMB and their data was deleted. Only real temporary uploads
  ("<name>.part-<8 hex>") are skipped now.
- prune refuses to run while a recorded per-user export's manifest is
  missing, instead of freeing that export's data.
- forget() and untrash() copy first, switch the catalog, and delete the
  originals last, so a crash can no longer leave the catalog pointing at a
  missing snapshot or database dump.
- Writes are checked against the expected size, so a short upload is an
  error instead of a truncated pack; raw restores check rename() and touch().
- A catalog write refuses to overwrite a generation another process just
  wrote, and the catalog lock lasts 300 s, so opening a large repository
  inside it no longer lets the lease run out.

Backup runs and scheduling:
- A file that keeps changing is read again at most three times; then the
  last read is kept and reported, instead of restarting forever and
  blocking cleanup.
- Unreadable files and folders are recorded, logged, shown by occ
  backup:run, stored in the snapshot and notified to admins, instead of
  being skipped silently.
- After a failed scheduled backup, at most three attempts a day, an hour
  apart, instead of a new run and notification every five minutes.
- The job no longer spins on a run whose lease another process holds, and
  the weekly cleanup catches up after a Sunday spent backing up.
- Notifications for the per-user export retention events.
This commit is contained in:
Dillard Blom
2026-10-06 23:48:40 +02:00
parent f39ca71dab
commit 0f4a238414
16 changed files with 319 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Repository/Repository.php
+28 -9
View File
@@ -236,8 +236,13 @@ final class Repository {
unlink($tmp);
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
}
rename($tmp, $dest);
touch($dest, $entry['m']);
if (!rename($tmp, $dest)) {
@unlink($tmp);
throw new RepositoryException('Cannot put the restored file in place: ' . $entry['p']);
}
if (!touch($dest, $entry['m'])) {
throw new RepositoryException('Cannot set the modification time of ' . $entry['p']);
}
$count++;
}
return $count;
@@ -375,28 +380,39 @@ final class Repository {
// must be decrypted and re-encrypted under the new path, not moved as raw bytes.
$dbInTrash = 'trash/' . $meta['db'];
$this->putObject($dbInTrash, $this->getObject($meta['db']));
$this->backend->delete($meta['db']);
}
// The snapshot object is bound to its id (not its path), so it can be moved as raw bytes.
$this->backend->move('snapshots/' . $snapshotId, 'trash/snapshots/' . $snapshotId);
// Copy first, switch the catalog, delete the originals last: a crash at any point leaves
// either the old state or the new one plus unreferenced leftovers, never a catalog entry
// pointing at a missing object. The snapshot object is bound to its id (not its path),
// so it can be copied as raw bytes.
$this->backend->put('trash/snapshots/' . $snapshotId, $this->backend->get('snapshots/' . $snapshotId));
$this->putObject('trash/info/' . $snapshotId, json_encode(['forgottenAt' => time(), 'by' => $by, 'db' => $meta['db'] ?? null,
'dbInTrash' => $dbInTrash, 'time' => $meta['time'] ?? null, 'label' => $meta['label'] ?? ''], JSON_THROW_ON_ERROR));
unset($this->catalog->snapshots[$snapshotId]);
$this->catalog->trash[$snapshotId] = true;
$this->catalog->write();
if ($dbInTrash !== null) {
$this->backend->delete($meta['db']);
}
$this->backend->delete('snapshots/' . $snapshotId);
}
/** Undo forget(): move a snapshot back from the trash. */
public function untrash(string $snapshotId): void {
$info = $this->trashInfo($snapshotId);
if ($info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash'])) {
$restoreDb = $info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash']);
if ($restoreDb) {
$this->putObject($info['db'], $this->getObject($info['dbInTrash']));
$this->backend->delete($info['dbInTrash']);
}
$this->backend->move('trash/snapshots/' . $snapshotId, 'snapshots/' . $snapshotId);
// Same order as forget(): copy, switch the catalog, then delete the trash copies.
$this->backend->put('snapshots/' . $snapshotId, $this->backend->get('trash/snapshots/' . $snapshotId));
$this->catalog->snapshots[$snapshotId] = true;
unset($this->catalog->trash[$snapshotId]);
$this->catalog->write();
if ($restoreDb) {
$this->backend->delete($info['dbInTrash']);
}
$this->backend->delete('trash/snapshots/' . $snapshotId);
$this->backend->delete('trash/info/' . $snapshotId);
}
@@ -530,7 +546,10 @@ final class Repository {
}
}
}
foreach ($this->backend->list('users') as $path) {
// listUserExports() throws when a recorded export's manifest is missing: then stop, since
// freeing its blobs would turn a tampering or storage alarm into irreversible deletion.
$exports = array_unique(array_merge($this->listUserExports(), $this->backend->list('users')));
foreach ($exports as $path) {
foreach (json_decode($this->getObject($path), true, 512, JSON_THROW_ON_ERROR)['entries'] as $e) {
foreach ($e['b'] ?? [] as $b) {
$used[$b] = true;