diff --git a/appinfo/info.xml b/appinfo/info.xml
index 105ab81..9b0bf02 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -5,7 +5,7 @@
NG Backup
Encrypted, incremental backups of your Nextcloud with restore from the web interface
- 0.0.16
+ 0.0.17
agpl
Dillard Blom
NgBackup
diff --git a/lib/Migration/Version000005Date20261002040000.php b/lib/Migration/Version000005Date20261002040000.php
new file mode 100644
index 0000000..f7a2dbe
--- /dev/null
+++ b/lib/Migration/Version000005Date20261002040000.php
@@ -0,0 +1,33 @@
+hasTable('ngb_leases')) {
+ return null;
+ }
+ $t = $schema->createTable('ngb_leases');
+ $t->addColumn('id', Types::STRING, ['notnull' => true, 'length' => 32]);
+ $t->addColumn('name', Types::STRING, ['notnull' => true, 'length' => 128]);
+ $t->addColumn('mode', Types::STRING, ['notnull' => true, 'length' => 1]);
+ $t->addColumn('expires_at', Types::BIGINT, ['notnull' => true, 'unsigned' => true]);
+ $t->setPrimaryKey(['id']);
+ $t->addIndex(['name'], 'ngb_leases_name');
+ return $schema;
+ }
+}
diff --git a/lib/Service/BackupService.php b/lib/Service/BackupService.php
index c7b1a9a..d195801 100644
--- a/lib/Service/BackupService.php
+++ b/lib/Service/BackupService.php
@@ -42,6 +42,7 @@ class BackupService {
private ILockingProvider $locking,
private LoggerInterface $logger,
private AlertService $alerts,
+ private LeaseService $leases,
) {
}
@@ -54,16 +55,15 @@ class BackupService {
}
// Check and create the run under the location's exclusive lock, so a cleanup cannot start
// in between (it takes the same lock and refuses while a run exists).
- $lock = PruneService::lockKey($target);
- try {
- $this->locking->acquireLock($lock, ILockingProvider::LOCK_EXCLUSIVE);
- } catch (LockedException) {
- throw new \RuntimeException('A cleanup or another start for ' . $target->getName() . ' is running; try again later');
+ // A run that is already in progress is simply continued; no lease needed for that.
+ $running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
+ if ($running !== []) {
+ return $running[0];
}
try {
- return $this->createRun($target, $label);
- } finally {
- $this->locking->releaseLock($lock, ILockingProvider::LOCK_EXCLUSIVE);
+ return $this->leases->with(PruneService::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $this->createRun($target, $label));
+ } catch (LockedException) {
+ throw new \RuntimeException('A cleanup or another start for ' . $target->getName() . ' is running; try again later');
}
}
@@ -103,18 +103,18 @@ class BackupService {
* run locked and returns immediately.
*/
public function step(Run $run, float $deadline): Run {
- $lockKey = 'ng_backup/run/' . $run->getId();
+ // Leases expire on their own: a step killed by a time limit or OOM blocks nothing for long.
+ $ttl = (int)min(3600, max(120, $deadline - microtime(true) + 120));
try {
- $this->locking->acquireLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
+ $runLease = $this->leases->acquire('run/' . $run->getId(), LeaseService::EXCLUSIVE, $ttl);
} catch (LockedException) {
- return $run;
+ return $run; // another process is doing this step
}
- // Shared lock on the location while this step writes: a cleanup (exclusive) cannot run now.
- $targetLock = 'ng_backup/target/' . $run->getTargetId();
+ // Shared lease on the location while this step writes: a cleanup (exclusive) cannot run now.
try {
- $this->locking->acquireLock($targetLock, ILockingProvider::LOCK_SHARED);
+ $targetLease = $this->leases->acquire('ng_backup/target/' . $run->getTargetId(), LeaseService::SHARED, $ttl);
} catch (LockedException) {
- $this->locking->releaseLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
+ $this->leases->release($runLease);
return $run;
}
try {
@@ -171,15 +171,22 @@ class BackupService {
}
return $run;
} finally {
- $this->locking->releaseLock($targetLock, ILockingProvider::LOCK_SHARED);
- $this->locking->releaseLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
+ $this->leases->release($targetLease);
+ $this->leases->release($runLease);
}
}
/** Run to completion in this process (occ, system cron), reporting progress. */
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
while ($run->getStatus() === Run::RUNNING) {
+ $before = $run->getUpdatedAt();
$run = $this->step($run, microtime(true) + $stepSeconds);
+ if ($run->getStatus() === Run::RUNNING && $run->getUpdatedAt() === $before) {
+ // Step was not possible (another process holds the run, or a crashed step's lease has
+ // not expired yet): wait instead of spinning.
+ sleep(5);
+ $run = $this->runs->find($run->getId());
+ }
if ($progress !== null) {
$progress($run);
}
diff --git a/lib/Service/LeaseService.php b/lib/Service/LeaseService.php
new file mode 100644
index 0000000..3609365
--- /dev/null
+++ b/lib/Service/LeaseService.php
@@ -0,0 +1,88 @@
+locking->acquireLock($guard, ILockingProvider::LOCK_EXCLUSIVE);
+ try {
+ $now = time();
+ $qb = $this->db->getQueryBuilder();
+ $qb->delete('ngb_leases')
+ ->where($qb->expr()->eq('name', $qb->createNamedParameter($name)))
+ ->andWhere($qb->expr()->lt('expires_at', $qb->createNamedParameter($now)));
+ $qb->executeStatement();
+
+ $qb = $this->db->getQueryBuilder();
+ $qb->select('mode')->from('ngb_leases')->where($qb->expr()->eq('name', $qb->createNamedParameter($name)));
+ $modes = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
+ if ($modes !== [] && ($mode === self::EXCLUSIVE || in_array(self::EXCLUSIVE, $modes, true))) {
+ throw new LockedException($name);
+ }
+ $id = bin2hex(random_bytes(16));
+ $qb = $this->db->getQueryBuilder();
+ $qb->insert('ngb_leases')->values([
+ 'id' => $qb->createNamedParameter($id),
+ 'name' => $qb->createNamedParameter($name),
+ 'mode' => $qb->createNamedParameter($mode),
+ 'expires_at' => $qb->createNamedParameter($now + $ttl),
+ ]);
+ $qb->executeStatement();
+ return $id;
+ } finally {
+ $this->locking->releaseLock($guard, ILockingProvider::LOCK_EXCLUSIVE);
+ }
+ }
+
+ /** Extend a lease while its holder is still working. */
+ public function refresh(string $id, int $ttl): void {
+ $qb = $this->db->getQueryBuilder();
+ $qb->update('ngb_leases')->set('expires_at', $qb->createNamedParameter(time() + $ttl))
+ ->where($qb->expr()->eq('id', $qb->createNamedParameter($id)));
+ $qb->executeStatement();
+ }
+
+ public function release(string $id): void {
+ $qb = $this->db->getQueryBuilder();
+ $qb->delete('ngb_leases')->where($qb->expr()->eq('id', $qb->createNamedParameter($id)));
+ $qb->executeStatement();
+ }
+
+ /** Run $fn under a lease; $refresh() can be called during long work. */
+ public function with(string $name, string $mode, int $ttl, callable $fn): mixed {
+ $id = $this->acquire($name, $mode, $ttl);
+ try {
+ return $fn(fn () => $this->refresh($id, $ttl));
+ } finally {
+ $this->release($id);
+ }
+ }
+}
diff --git a/lib/Service/PruneService.php b/lib/Service/PruneService.php
index 5ae078c..478591c 100644
--- a/lib/Service/PruneService.php
+++ b/lib/Service/PruneService.php
@@ -29,15 +29,16 @@ class PruneService {
private ILockingProvider $locking,
private AlertService $alerts,
private KeyService $keys,
+ private LeaseService $leases,
) {
}
public function untrash(Target $target, string $snapshotId): void {
- $this->lockExclusive($target);
+ $lease = $this->lockExclusive($target, 300);
try {
$this->doUntrash($target, $snapshotId);
} finally {
- $this->locking->releaseLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
+ $this->leases->release($lease);
}
}
@@ -47,31 +48,27 @@ class PruneService {
*/
public function underStartLock(Target $target, callable $fn): mixed {
try {
- $this->locking->acquireLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
+ return $this->leases->with(self::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $fn());
} catch (LockedException) {
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
}
- try {
- return $fn();
- } finally {
- $this->locking->releaseLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
- }
}
/**
* Exclusive lock on the location, then make sure no run of any kind (backup, restore, export)
* is in progress: their data may not be referenced by a snapshot yet, or is being read.
*/
- private function lockExclusive(Target $target): void {
+ private function lockExclusive(Target $target, int $ttl): string {
try {
- $this->locking->acquireLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
+ $lease = $this->leases->acquire(self::lockKey($target), LeaseService::EXCLUSIVE, $ttl);
} catch (LockedException) {
throw new \RuntimeException('A backup, restore or cleanup of ' . $target->getName() . ' is in progress; try again later');
}
if ($this->runs->findRunning($target->getId()) !== []) {
- $this->locking->releaseLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
+ $this->leases->release($lease);
throw new \RuntimeException('A backup or restore of ' . $target->getName() . ' is running; try again later');
}
+ return $lease;
}
private function doUntrash(Target $target, string $snapshotId): void {
@@ -126,7 +123,7 @@ class PruneService {
if ($target->getAppendOnly()) {
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
}
- $this->lockExclusive($target);
+ $lease = $this->lockExclusive($target, 1800);
try {
$repo = $this->targets->repository($target);
$times = [];
@@ -160,7 +157,7 @@ class PruneService {
$prune = $repo->prune($dryRun);
return ['kept' => $keep, 'forgotten' => $forget, 'purged' => $purged, 'prune' => $prune];
} finally {
- $this->locking->releaseLock(self::lockKey($target), ILockingProvider::LOCK_EXCLUSIVE);
+ $this->leases->release($lease);
}
}
}
diff --git a/lib/Service/RestoreService.php b/lib/Service/RestoreService.php
index 1750a1c..95a1d95 100644
--- a/lib/Service/RestoreService.php
+++ b/lib/Service/RestoreService.php
@@ -36,23 +36,17 @@ class RestoreService {
public function __construct(
private TargetService $targets,
private IRootFolder $root,
- private \OCP\Lock\ILockingProvider $locking,
+ private LeaseService $leases,
) {
}
/** Shared lock on the location while restoring: a cleanup (exclusive) must not delete what is being read. */
- private function withSharedLock(Target $target, callable $fn): mixed {
- $key = PruneService::lockKey($target);
+ private function withSharedLock(Target $target, callable $fn, int $ttl = 600): mixed {
try {
- $this->locking->acquireLock($key, \OCP\Lock\ILockingProvider::LOCK_SHARED);
+ return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, $fn);
} catch (\OCP\Lock\LockedException) {
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is running; try again later');
}
- try {
- return $fn();
- } finally {
- $this->locking->releaseLock($key, \OCP\Lock\ILockingProvider::LOCK_SHARED);
- }
}
/**
@@ -88,7 +82,20 @@ class RestoreService {
* @return array{restored:int, trashed:int, target:string}
*/
public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array {
- return $this->withSharedLock($target, fn () => $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $progress));
+ return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress) {
+ $last = time();
+ // Keep the lease alive while files are being written.
+ $beat = function (int $done, int $total, string $name) use ($refresh, $progress, &$last): void {
+ if (time() - $last >= 60) {
+ $refresh();
+ $last = time();
+ }
+ if ($progress !== null) {
+ $progress($done, $total, $name);
+ }
+ };
+ return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat);
+ });
}
private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress): array {
@@ -150,7 +157,7 @@ class RestoreService {
if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) {
throw new \RuntimeException("Cannot create $directory");
}
- return $this->withSharedLock($target, fn () => $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/'));
+ return $this->withSharedLock($target, fn () => $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/'), 3600);
}
private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void {