Compare commits
109 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 89a126999f | |||
| 1b986c9d31 | |||
| 025bbfbb62 | |||
| 8ed7455619 | |||
| 30a250bc11 | |||
| 9c06ab2841 | |||
| d659322e60 | |||
| 7726c179f0 | |||
| df732f185a | |||
| b3c8068f8d | |||
| 0a9bd963fb | |||
| 379b81db02 | |||
| 828f3f142a | |||
| c146d700e8 | |||
| 8e273ac259 | |||
| 6385925845 | |||
| 894258f836 | |||
| 54750e9742 | |||
| fe92ef70bf | |||
| b3e3054737 | |||
| c775e27dd2 | |||
| d8e772137f | |||
| bcfec17cfe | |||
| b309081d1a | |||
| 5f8d5faee7 | |||
| 65f14b4380 | |||
| 4fa7cbf6ef | |||
| f4abfe0a8d | |||
| fe1a289c2d | |||
| ab3265e42c | |||
| 466cff8857 | |||
| 6bba0b59fc | |||
| a01aa16ccd | |||
| cc20b9174f | |||
| 239aa1b923 | |||
| 62766e1e71 | |||
| 9775cdc475 | |||
| a8da8cb54a | |||
| bfd54447b0 | |||
| 0b50bb6b81 | |||
| f415bab519 | |||
| 27eb0b32fc | |||
| b1d4f6a469 | |||
| fe7ad9b4b5 | |||
| 695d972785 | |||
| 972a52e665 | |||
| 7c08a10621 | |||
| 209520165e | |||
| 9554e774e5 | |||
| 8b2dca28e3 | |||
| 5742f77253 | |||
| 550249570b | |||
| 0eb23f56e7 | |||
| a0c1f1cd58 | |||
| 26cc9a897e | |||
| fe0385ad8c | |||
| b37a953cfd | |||
| b89b5b5fa6 | |||
| 16e09f4fde | |||
| 45069994e9 | |||
| 0f4a238414 | |||
| f39ca71dab | |||
| e8ecb77df5 | |||
| 70965dd4bd | |||
| cc09d74942 | |||
| eb9100a74d | |||
| 614caf9f63 | |||
| 943c95bf07 | |||
| ba7f70e0f8 | |||
| d70a869070 | |||
| c9f82a79e4 | |||
| 932f60bc5e | |||
| 935c7b6a08 | |||
| f18b4f0862 | |||
| 6cbc5d79d4 | |||
| 58711b67ef | |||
| e7229626f8 | |||
| 807353cbe4 | |||
| 48ea6bb33d | |||
| 9d97f9c1f1 | |||
| b6ba09b078 | |||
| 85e835bfc2 | |||
| 2610c1ad2f | |||
| 532b23482e | |||
| 066873a818 | |||
| c54b2cebed | |||
| 48bb829d45 | |||
| f75ed3aed6 | |||
| 131bb6178d | |||
| ac82b38e37 | |||
| d3aa07b106 | |||
| b9e8858d76 | |||
| 38654317a2 | |||
| fabdfa6786 | |||
| 36e6a3fde7 | |||
| c601a02b2b | |||
| 370f3c55fa | |||
| 3f6785bcc0 | |||
| 4a28bf8790 | |||
| 506854d597 | |||
| 9bfb6ae334 | |||
| 7b33347dbc | |||
| 06e55b08ea | |||
| 93fec0e8f3 | |||
| d474050be0 | |||
| bc16a186d5 | |||
| 915e534ab2 | |||
| 1839997972 | |||
| 07e3d263b4 |
@@ -0,0 +1,106 @@
|
|||||||
|
# SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
unit:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
php: ['8.2', '8.3', '8.4']
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- uses: shivammathur/setup-php@v2
|
||||||
|
with:
|
||||||
|
php-version: ${{ matrix.php }}
|
||||||
|
extensions: sodium, zip
|
||||||
|
coverage: none
|
||||||
|
- run: composer install --no-interaction --no-progress
|
||||||
|
- run: vendor/bin/phpunit -c tests/phpunit.xml
|
||||||
|
|
||||||
|
psalm:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- uses: shivammathur/setup-php@v2
|
||||||
|
with:
|
||||||
|
php-version: '8.2'
|
||||||
|
extensions: sodium, zip
|
||||||
|
coverage: none
|
||||||
|
- run: composer install --no-interaction --no-progress
|
||||||
|
- run: composer psalm
|
||||||
|
|
||||||
|
integration:
|
||||||
|
# Full round trip (key, kit, location, backup, incremental, restore modes, database dump and
|
||||||
|
# restore, retention, trash) on a fresh Nextcloud per version and database.
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
nextcloud: ['32', '33', '34', '35']
|
||||||
|
database: ['sqlite', 'mariadb', 'pgsql']
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: Round trip on Nextcloud ${{ matrix.nextcloud }} / ${{ matrix.database }}
|
||||||
|
run: tests/integration/matrix.sh "${{ matrix.nextcloud }}" "${{ matrix.database }}"
|
||||||
|
|
||||||
|
integration-targets:
|
||||||
|
# Same round trip, but through the real S3 (s3mock) and SFTP backends instead of a local
|
||||||
|
# path, so files_external's AmazonS3/SFTP backend and auth classes are actually exercised.
|
||||||
|
# Backend correctness does not depend on the Nextcloud version or database, so this runs once.
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: Round trip against S3 and SFTP
|
||||||
|
run: tests/integration/targets-matrix.sh
|
||||||
|
|
||||||
|
integration-user:
|
||||||
|
# Per-user backup/restore via user_migration (occ backup:user:backup / backup:user:restore),
|
||||||
|
# an optional dependency installed from the app store. It calls user_migration's internal
|
||||||
|
# UserMigrationService, whose API can change between Nextcloud versions, so this runs on every
|
||||||
|
# supported version (not the database, which the main integration job already covers).
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
nextcloud: ['32', '33', '34', '35']
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: Set up a fresh Nextcloud instance
|
||||||
|
run: |
|
||||||
|
docker run -d --name ngbu -e SQLITE_DATABASE=nextcloud -e NEXTCLOUD_ADMIN_USER=admin \
|
||||||
|
-e NEXTCLOUD_ADMIN_PASSWORD=admin-user-roundtrip -e NEXTCLOUD_TRUSTED_DOMAINS=localhost \
|
||||||
|
-v "$PWD:/var/www/html/custom_apps/ng_backup:ro" nextcloud:${{ matrix.nextcloud }}-apache
|
||||||
|
for i in $(seq 1 90); do docker exec -u www-data ngbu php occ status 2>/dev/null | grep -q "installed: true" && break; sleep 4; done
|
||||||
|
docker exec ngbu chown www-data:www-data /var/www/html/custom_apps
|
||||||
|
- name: Per-user backup/restore round trip on Nextcloud ${{ matrix.nextcloud }}
|
||||||
|
run: tests/integration/user-roundtrip.sh "docker exec -u www-data ngbu" /var/www/html/data
|
||||||
|
|
||||||
|
integration-full-restore:
|
||||||
|
# Disaster recovery (occ backup:restore:full) on a SECOND, fresh Nextcloud instance that
|
||||||
|
# never saw the first one, sharing only the backup location. Does not depend on the database
|
||||||
|
# beyond what the main integration job already covers, so this runs once, on one NC version.
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: Full restore round trip
|
||||||
|
run: tests/integration/full-restore-roundtrip.sh 35-apache
|
||||||
+8
-3
@@ -1,5 +1,10 @@
|
|||||||
/vendor/
|
|
||||||
/node_modules/
|
|
||||||
/js/
|
|
||||||
/build/
|
/build/
|
||||||
|
cred-*
|
||||||
|
*credentials*
|
||||||
|
/css/
|
||||||
|
/js/
|
||||||
|
/node_modules/
|
||||||
|
.phpunit.result.cache
|
||||||
|
*.secret
|
||||||
/spikes/out/
|
/spikes/out/
|
||||||
|
/vendor/
|
||||||
|
|||||||
+225
@@ -0,0 +1,225 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to NG Backup are documented here. The format follows
|
||||||
|
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows the stages in
|
||||||
|
`PLAN.md` section 9. The `<version>` in `appinfo/info.xml` is plain SemVer-numeric on purpose:
|
||||||
|
the app store accepts a `-beta` suffix, but Nextcloud servers then only offer the release on the
|
||||||
|
beta/daily update channel, which would hide the app from most admins. The stability stage is
|
||||||
|
instead carried by the app description, the git tag, the release, and this file.
|
||||||
|
|
||||||
|
## [0.9.5-beta.6] - 2026-10-12
|
||||||
|
|
||||||
|
Bugfix release for SFTP locations and large repositories. Tested on a production instance with a
|
||||||
|
1 TB repository on a Hetzner Storage Box over SFTP: a check in 5.5 minutes, a deep check of all
|
||||||
|
316,529 blobs (815 GiB) in 11 hours, and an incremental backup in under 8 minutes.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Progress of a running backup in the web interface and `occ backup:status`, e.g.
|
||||||
|
"running (files) 68%". The total is the size of the previous snapshot; for a first backup it
|
||||||
|
is estimated from Nextcloud's file cache (users' files, versions and trash) and shown as
|
||||||
|
"~68%".
|
||||||
|
- Edit a backup location (web interface and `occ backup:target:edit`): change the password, host,
|
||||||
|
folder, connection type or name without losing its history. Before this the only option was a
|
||||||
|
new location, which starts an empty repository and, with a retention lock at the provider,
|
||||||
|
keeps paying for the old one. The new settings are only saved when they lead to the same
|
||||||
|
repository; stored passwords are never sent to the browser, and an empty password field keeps
|
||||||
|
the stored one. Administrators are notified of the change.
|
||||||
|
- `occ backup:verify` reports how far it is every 30 seconds, and `occ backup:status` shows the
|
||||||
|
progress of a running check and how the last check of each location ended. On a large
|
||||||
|
repository a check takes hours, a deep one days.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- Checking and opening a large repository over SFTP is much faster: the index files (one per
|
||||||
|
pack, 27,547 for 1 TB) are downloaded 32 at a time instead of one after the other, 114 instead
|
||||||
|
of 6 files per second to a Hetzner Storage Box (about 4 minutes instead of 77 for 1 TB). A check
|
||||||
|
of a repository with more than 256 packs also lists the pack folders (256 requests) instead of
|
||||||
|
asking for every pack.
|
||||||
|
- App store listing: the author's e-mail address is shown.
|
||||||
|
- App store listing: the homepage link points to the GitHub repository instead of back to the
|
||||||
|
app store page itself.
|
||||||
|
- The notification for a location without a successful backup read "No successful backup to …
|
||||||
|
since never" when there had never been one; it now reads "No successful backup to … yet".
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- 0.9.4 could not open a repository at the root of an SFTP location (empty path): its faster
|
||||||
|
SFTP reads sent their first request before phpseclib had opened the SFTP channel, which only
|
||||||
|
happens on the first SFTP call after login. Every backup then failed with "No repository at
|
||||||
|
this location". The channel is now opened first.
|
||||||
|
- A check (`occ backup:verify`) did not renew its hold on the location while it downloaded the
|
||||||
|
index; on a large repository that takes longer than the hold lasts, so a weekly cleanup could
|
||||||
|
have started during the check.
|
||||||
|
- A long first backup could end as failed although its snapshot was complete: when its final
|
||||||
|
step took longer than the step's lease (150 s with `occ backup:run`), the background job took
|
||||||
|
the run over, finished it a second time and failed on the snapshot it had already recorded,
|
||||||
|
overwriting the result. The lease now lasts at least 15 minutes, a step writes the run only if
|
||||||
|
no other process wrote it since the step loaded it (in one database statement), and recording
|
||||||
|
a snapshot that is already recorded is no longer an error.
|
||||||
|
- `occ backup:verify` on a large repository failed with "SFTP connection failed while opening
|
||||||
|
snapshots/…" or "Not found: snapshots/…". A Hetzner Storage Box drops an SFTP connection that
|
||||||
|
carried a lot of data (some 40 MB) and then stayed quiet for a few minutes, and the check went
|
||||||
|
quiet after downloading the index: it refreshed the local index cache, first in 29 minutes (a
|
||||||
|
pass over all blobs for every pack: 27,547 packs, 400,000 blobs for 1 TB), then still in more
|
||||||
|
than 6 minutes (a database transaction per pack on PostgreSQL). Nextcloud's SFTP storage keeps
|
||||||
|
using a dropped connection, on which every file looks missing. Now:
|
||||||
|
- an SFTP connection that was not used for a minute is checked first, and one that turns out
|
||||||
|
to be dead after a failed request is replaced, once per request; uploads, moves and deletes
|
||||||
|
are only checked first, never repeated;
|
||||||
|
- the index cache is written in one transaction (27,547 packs in 11 seconds on a test server,
|
||||||
|
against about a minute row by row);
|
||||||
|
- after a check the cache holds the index files as they are: before, a blob that was in two
|
||||||
|
packs was only cached for one of them.
|
||||||
|
|
||||||
|
## [0.9.4-beta.5] - 2026-10-09
|
||||||
|
|
||||||
|
Bugfix release: faster restores.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Restores and `occ backup:verify --deep` fetched every blob with its own request, so on a
|
||||||
|
remote location they ran far below the speed of the connection. The blobs of a pack are now
|
||||||
|
read from one open stream, in the order they were written. Restoring 86 MB in 202 files from a
|
||||||
|
Hetzner Storage Box over SMB went from 0.78 MB/s to 4.2 MB/s, the speed of reading the packs
|
||||||
|
as plain files over that connection.
|
||||||
|
- Reading from an SFTP location waited a full round trip for every 256 KiB, whatever the line
|
||||||
|
speed. Reads now keep up to 8 requests in flight. Restoring 1.97 GB in 500 files from a Hetzner
|
||||||
|
Storage Box (47 ms away) took 694 s before the fix above, 499 s with it and 142 s with both
|
||||||
|
(2.8 → 3.9 → 13.9 MB/s); a full 51 GB restore ran at 13.0 MB/s. When a server error
|
||||||
|
interrupts such a read, it goes on from the same position with Nextcloud's own SFTP stream;
|
||||||
|
after three failing files in a row, or on a Nextcloud whose SFTP library lacks what this needs,
|
||||||
|
only Nextcloud's own stream is used.
|
||||||
|
- README: a WebDAV location on a slow upload needs a higher `davstorage.request_timeout`.
|
||||||
|
|
||||||
|
## [0.9.3-beta.4] - 2026-10-08
|
||||||
|
|
||||||
|
Bugfix release.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- A backup failed with "cannot execute INSERT in a read-only transaction" on PostgreSQL as soon
|
||||||
|
as the database dump filled a whole pack (32 MiB): the pack's index cache row was written
|
||||||
|
inside the read-only dump transaction. On other databases that row was rolled back silently.
|
||||||
|
- Backups to a remote location slowed down as the repository grew: every backup step listed the
|
||||||
|
repository's index with a round trip per file. Over SFTP to a Hetzner Storage Box a 60 GiB
|
||||||
|
repository already lost a large part of each step to it; backing up continuously
|
||||||
|
(`occ backup:run`) went from 45 KiB/s to 5.8 MB/s with the fix.
|
||||||
|
|
||||||
|
## [0.9.2-beta.3] - 2026-10-08
|
||||||
|
|
||||||
|
Bugfix release.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- README: notes on using a Hetzner Storage Box over WebDAV and SMB.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Adding a backup location in the web interface failed with "Request failed with status code 403"
|
||||||
|
when the login was more than 30 minutes old: the form did not ask for password confirmation.
|
||||||
|
- Settings page: the "Backup and restore" / "Settings" tabs line up with the sections below them.
|
||||||
|
- Backups: the runs table scrolls horizontally on narrow screens instead of running off the page.
|
||||||
|
- `occ backup:verify --deep`: a blob that could not be read (network or storage error) was reported
|
||||||
|
as "corrupted or tampered". Reads are now retried, and a read that keeps failing is reported
|
||||||
|
separately as unreadable, with the error (run the check again; if the same blobs stay unreadable,
|
||||||
|
their pack may be damaged or truncated). Only a blob that was read but fails decryption or its
|
||||||
|
hash check counts as corrupted. Found on a Hetzner Storage Box over WebDAV, where the blob
|
||||||
|
turned out to be intact.
|
||||||
|
|
||||||
|
## [0.9.1-beta.2] - 2026-10-07
|
||||||
|
|
||||||
|
Second beta, the first release in the Nextcloud app store (signed).
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- App store listing: screenshots, website, repository and documentation link in `appinfo/info.xml`.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Restore: the "new folder" option showed `"Restored "` without the date pattern.
|
||||||
|
- Restore: the restore-point dropdown cut off the time.
|
||||||
|
- Settings: the passphrase actions are on one line instead of stacked.
|
||||||
|
- `docs/HOWTO.md`: the SFTP example used a non-existent auth mechanism (`sftp::password`
|
||||||
|
instead of `password::password`).
|
||||||
|
|
||||||
|
## [0.9.0-beta.1] - 2026-10-07
|
||||||
|
|
||||||
|
First beta. Tested thoroughly and usable **alongside** your current backup, but **not yet a
|
||||||
|
drop-in replacement** for it. We test what we can ourselves (Nextcloud 32–35, SQLite, MariaDB and
|
||||||
|
PostgreSQL, S3, SFTP, FTP, SMB and NFS), but we need field experience from other setups: please
|
||||||
|
report what works and what doesn't, especially on large instances and on storage providers not
|
||||||
|
listed in the README.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Disaster recovery: `occ backup:key:import` and `occ backup:restore:full` restore the database,
|
||||||
|
data directory and config onto a fresh installation of the same major version.
|
||||||
|
- Per-user backup and restore via user_migration: `occ backup:user:backup` and
|
||||||
|
`occ backup:user:restore` (replace the account after a safety export, or restore as
|
||||||
|
`<uid>-bak`), also onto a server where the user does not exist yet. The commands ask before
|
||||||
|
installing user_migration (or take `--install-user-migration`).
|
||||||
|
- Checksum audit: `occ backup:verify`, shallow or `--deep`, for snapshots and per-user exports.
|
||||||
|
- Retention for per-user exports: `occ backup:retention --user-last=N`.
|
||||||
|
- FTP and SMB covered by the integration tests; NFS through the `local` backend on a mounted share.
|
||||||
|
- `docs/HOWTO.md`: step by step from setup to a tested restore.
|
||||||
|
- Warnings when a backup finishes with unreadable files or folders, or with files that kept
|
||||||
|
changing while being read (log, `occ`, notification, snapshot).
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- `backup:restore:full` refuses on an installation with more than one user and asks first;
|
||||||
|
`backup:user:restore --mode=replace` asks before deleting an existing account (`--force` for
|
||||||
|
scripts in both cases).
|
||||||
|
- A failed scheduled backup is retried at most three times a day, an hour apart.
|
||||||
|
- The recovery kit carries the deletion delay; the kit file is created owner-only.
|
||||||
|
- Adding a location through the web API needs password confirmation.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Exports of users whose id contains `.part-` could be deleted by cleanup on remote locations.
|
||||||
|
- Cleanup no longer frees the data of a per-user export whose manifest is missing.
|
||||||
|
- Crash safety of moving snapshots to and from the trash, of catalog writes by concurrent
|
||||||
|
backups and per-user exports, and of short uploads.
|
||||||
|
- A file that keeps changing no longer makes a backup restart forever.
|
||||||
|
- Long restores and exports keep their lock on the location alive.
|
||||||
|
- Floating-point database columns are dumped exactly.
|
||||||
|
|
||||||
|
### Known limitations
|
||||||
|
See "Known limitations" in the README: memory use on very large instances, never two servers
|
||||||
|
writing to one location, removing a passphrase slot does not revoke access, no rollback detection
|
||||||
|
before a disaster recovery, and a few smaller points. None of them block the road to 1.0.0; they
|
||||||
|
are on the backlog in `PLAN.md`.
|
||||||
|
|
||||||
|
## [0.1.0-alpha.1] - 2026-10-05
|
||||||
|
|
||||||
|
First public alpha. Feature-complete for phase 1: scheduled, encrypted, incremental backups, and
|
||||||
|
restore of files and folders. **Not for production use** — test on a copy of production (dev,
|
||||||
|
test, staging). Full disaster recovery (`backup:restore:full` on a fresh installation) is phase 2
|
||||||
|
and not implemented yet.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Encrypted (XChaCha20-Poly1305), content-addressed, deduplicated, incremental backups, streamed
|
||||||
|
from source to destination without a local copy of the data.
|
||||||
|
- Backup locations via files_external's own backend/auth classes, without ever mounting the
|
||||||
|
location: S3 (and compatible services), SFTP, WebDAV, FTP, Swift, or a local path.
|
||||||
|
- A recovery kit with up to three passphrase slots, mandatory download-and-confirm before the
|
||||||
|
first backup runs, and an optional encrypted copy by email.
|
||||||
|
- Resumable, time-boxed background jobs (database dump, file walk, upload), so a run survives
|
||||||
|
short PHP time limits and a crash is resumed, never restarted, without leaving the instance in
|
||||||
|
maintenance mode.
|
||||||
|
- Database dump and restore without external tools, via Nextcloud's own `IDBConnection`/DBAL,
|
||||||
|
streamed through the same pipeline as file data.
|
||||||
|
- Restore of files and folders via the web interface and `occ`, into a new folder, merged, or
|
||||||
|
replacing the original (with the previous content going to the trash).
|
||||||
|
- Retention policy (last/daily/weekly/monthly), prune with pack repacking, and a snapshot trash
|
||||||
|
with a configurable deletion delay.
|
||||||
|
- Append-only locations and documented recommendations against ransomware and a compromised
|
||||||
|
server (object lock, delete-less credentials, off-site copies).
|
||||||
|
- Admin settings page (locations, schedule, retention, key management) and a JSON API.
|
||||||
|
- Notifications and an admin setup check for a failed or overdue backup.
|
||||||
|
- CI: unit tests on PHP 8.2–8.4, and an end-to-end backup→restore round trip on Nextcloud 32–35 ×
|
||||||
|
SQLite/MariaDB/PostgreSQL, and against real S3 and SFTP backends (not just a local path).
|
||||||
|
|
||||||
|
### Known limitations
|
||||||
|
- No full/disaster-recovery restore yet (phase 2).
|
||||||
|
- No per-user backup/restore via user_migration yet (phase 2).
|
||||||
|
- No FTP/SMB coverage in CI yet, and no verification/checksum-audit command yet (phase 2).
|
||||||
|
- See the "Storage backend experiences" section of the README for provider-specific quirks found
|
||||||
|
so far (e.g. Hetzner Storage Box sub-accounts).
|
||||||
|
|
||||||
|
[0.9.5-beta.6]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.5-beta.6
|
||||||
|
[0.9.4-beta.5]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.4-beta.5
|
||||||
|
[0.9.3-beta.4]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.3-beta.4
|
||||||
|
[0.9.2-beta.3]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.2-beta.3
|
||||||
|
[0.9.1-beta.2]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.1-beta.2
|
||||||
|
[0.9.0-beta.1]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.0-beta.1
|
||||||
|
[0.1.0-alpha.1]: https://github.com/dillardblom/ng_backup/releases/tag/v0.1.0-alpha.1
|
||||||
@@ -214,8 +214,8 @@ Daarom:
|
|||||||
### 3.8 Backup en restore per gebruiker: via user_migration
|
### 3.8 Backup en restore per gebruiker: via user_migration
|
||||||
|
|
||||||
In plaats van zelf te bepalen welke gegevens van een gebruiker meegaan, hergebruiken we
|
In plaats van zelf te bepalen welke gegevens van een gebruiker meegaan, hergebruiken we
|
||||||
**user_migration** (officiële Nextcloud-app, actief onderhouden, NC29–36; staat ook op nc01).
|
**user_migration** (officiële Nextcloud-app, actief onderhouden, NC29–36; draait ook op onze eigen productie-instantie).
|
||||||
Die app werkt met *migrators* die andere apps zelf aanleveren. Op nc01 hebben die er al:
|
Die app werkt met *migrators* die andere apps zelf aanleveren. Op onze productie-instantie zijn dat er al:
|
||||||
calendar, dav (agenda's en contactpersonen), deck, files_trashbin, mail, settings (profiel,
|
calendar, dav (agenda's en contactpersonen), deck, files_trashbin, mail, settings (profiel,
|
||||||
account, app-instellingen) en de bestanden zelf. Elke app die later een migrator toevoegt, gaat
|
account, app-instellingen) en de bestanden zelf. Elke app die later een migrator toevoegt, gaat
|
||||||
automatisch mee.
|
automatisch mee.
|
||||||
@@ -276,7 +276,7 @@ onderdelen wel en niet terugkomen, op basis van de geregistreerde migrators.
|
|||||||
| `backup:list [--target=]` | herstelpunten |
|
| `backup:list [--target=]` | herstelpunten |
|
||||||
| `backup:verify <snapshot> [--deep]` | checksums controleren, optioneel alles ontsleutelen |
|
| `backup:verify <snapshot> [--deep]` | checksums controleren, optioneel alles ontsleutelen |
|
||||||
| `backup:restore:file <snapshot> <user> <pad> [--to=]` | bestand/map terugzetten |
|
| `backup:restore:file <snapshot> <user> <pad> [--to=]` | bestand/map terugzetten |
|
||||||
| `backup:restore:user <snapshot> <user> [--to-user=]` | gebruiker terugzetten via user_migration (fase 2) |
|
| `backup:user:restore <snapshot> <user> [--to-user=]` | gebruiker terugzetten via user_migration (fase 2) |
|
||||||
| `backup:restore:full <snapshot>` | volledige restore (fase 2, verse installatie) |
|
| `backup:restore:full <snapshot>` | volledige restore (fase 2, verse installatie) |
|
||||||
| `backup:target:add/list/remove/test` | opslagdoelen beheren en testen |
|
| `backup:target:add/list/remove/test` | opslagdoelen beheren en testen |
|
||||||
| `backup:key:export / import` | herstelkit |
|
| `backup:key:export / import` | herstelkit |
|
||||||
@@ -308,14 +308,14 @@ niet met de hunne, maar bij installatie waarschuwen als `backup` ook actief is.
|
|||||||
| 2 | Backup/restore per gebruiker via user_migration, volledige restore op verse installatie, FTP/SMB, verificatie | 3–4 weken |
|
| 2 | Backup/restore per gebruiker via user_migration, volledige restore op verse installatie, FTP/SMB, verificatie | 3–4 weken |
|
||||||
| 3 | In-place volledige restore (restore-modus), zelfbediening voor gebruikers, extra doelen | open |
|
| 3 | In-place volledige restore (restore-modus), zelfbediening voor gebruikers, extra doelen | open |
|
||||||
|
|
||||||
Release in de app store pas na fase 1 en een periode van eigen gebruik op nc01 (net als occweb):
|
Release in de app store pas na fase 1 en een periode van eigen gebruik op onze productie-instantie (net als occweb):
|
||||||
een backup-app die niet betrouwbaar is, is erger dan geen backup-app.
|
een backup-app die niet betrouwbaar is, is erger dan geen backup-app.
|
||||||
|
|
||||||
## 8. Besluiten (2026-10-01)
|
## 8. Besluiten (2026-10-01)
|
||||||
|
|
||||||
1. **Doelgroep:** alle varianten: AIO, losse installaties én gehoste Nextcloud zonder SSH.
|
1. **Doelgroep:** alle varianten: AIO, losse installaties én gehoste Nextcloud zonder SSH.
|
||||||
2. **Licentie:** AGPL-3.0-or-later.
|
2. **Licentie:** AGPL-3.0-or-later.
|
||||||
3. **Repository:** Ensembia op Gitea + GitHub (dillardblom), zoals occweb.
|
3. **Repository:** eigen Gitea + GitHub (dillardblom), zoals occweb.
|
||||||
4. **Naam:** "NG Backup", app-id `ng_backup`, occ-voorvoegsel `backup:`.
|
4. **Naam:** "NG Backup", app-id `ng_backup`, occ-voorvoegsel `backup:`.
|
||||||
5. **Herstelkit:** verplicht downloaden + expliciete bevestiging door de beheerder (vinkje +
|
5. **Herstelkit:** verplicht downloaden + expliciete bevestiging door de beheerder (vinkje +
|
||||||
"Yes, I confirm"), zie 3.2; versleuteld per e-mail als optionele extra kopie.
|
"Yes, I confirm"), zie 3.2; versleuteld per e-mail als optionele extra kopie.
|
||||||
@@ -325,6 +325,67 @@ een backup-app die niet betrouwbaar is, is erger dan geen backup-app.
|
|||||||
export, verwijderen, opnieuw aanmaken), óf terugzetten als `<gebruiker>-bak`, zie 3.8.
|
export, verwijderen, opnieuw aanmaken), óf terugzetten als `<gebruiker>-bak`, zie 3.8.
|
||||||
8. **Fase 0 afgerond (2026-10-01),** resultaten in `spikes/RESULTS.md`.
|
8. **Fase 0 afgerond (2026-10-01),** resultaten in `spikes/RESULTS.md`.
|
||||||
|
|
||||||
|
## 9. Releases en versies (besluit 2026-10-02)
|
||||||
|
|
||||||
|
| Versie | Betekenis |
|
||||||
|
|---|---|
|
||||||
|
| 0.1.0-alpha.N | publiek; meetesten gewaardeerd op een kopie van productie (dev, test, staging) |
|
||||||
|
| 0.9.0-beta.N | mag meedraaien op productie, naast de reguliere backup; volledige restore aanwezig en getest |
|
||||||
|
| 1.0.0-rc.N | releasekandidaten, alleen bugfixes |
|
||||||
|
| 1.0.0 | stabiel: vervanging van de reguliere backup, noodherstel bewezen |
|
||||||
|
| 1.2.0 e.v. | functies van fase 2 (restore per gebruiker, FTP/SMB, verificatie, ...) |
|
||||||
|
|
||||||
|
Volgorde: fase 1 grondig testen, reviewen en debuggen; dan nadenken over 0.1.0-alpha.1. Een
|
||||||
|
minimale volledige restore (`occ backup:restore:full` op een verse installatie) komt vóór de beta.
|
||||||
|
|
||||||
|
**Roadmap-indeling fase 2 (besluit 2026-10-05):** naar v1.2.0 toewerken met in fase 2:
|
||||||
|
1. `backup:restore:full` (disaster recovery, verse installatie) — **klaar** (2026-10-05)
|
||||||
|
2. FTP/SMB-ondersteuning — **klaar** (2026-10-06): beide liepen al via files_external's generieke
|
||||||
|
adapter zonder enige ng_backup-code; alleen testdekking toegevoegd
|
||||||
|
(`tests/integration/targets-matrix.sh`) + bevindingen in README ("Storage backend experiences").
|
||||||
|
SMB vereist de `smbclient` PECL-extensie (niet standaard in de Nextcloud-image, in CI nu
|
||||||
|
on-the-fly gebouwd). NFS is geen eigen backend: lokaal mounten + `local`-backend erop richten.
|
||||||
|
3. Verificatie/checksum-audit-commando — **klaar** (2026-10-05)
|
||||||
|
4. Retentie/prune voor user-exports — **klaar** (2026-10-06): `occ backup:retention --user-last=N`
|
||||||
|
(standaard 3, per installatie, net als de bestaande snapshot-policy), toegepast door
|
||||||
|
`occ backup:prune` (geen prullenbak voor user-exports, in tegenstelling tot snapshots — on-demand,
|
||||||
|
niet het primaire vangnet).
|
||||||
|
5. Automatische user-export in het reguliere schema — **naar backlog** (niet voor deze release; on-demand blijft)
|
||||||
|
9. SMB-roundtrip (protocol, niet een specifieke server) — **klaar** (2026-10-06): CI-matrix, 2 GB-roundtrip en
|
||||||
|
user-export via SMB; NFS via de OMV-share getest (`local`-backend op een gemounte share, 2026-10-06)
|
||||||
|
10. `backup:user:restore` zonder user_migration: vraagt om toestemming om de app te installeren en te activeren (of
|
||||||
|
`--install-user-migration`) — **klaar** (2026-10-06); waarschuwing over de settings-allowlist — **klaar**;
|
||||||
|
agenda/contacten bij migratie naar een server zonder die gebruiker — **getest** (2026-10-06, naar `migrated-personal`/`migrated-contacts`)
|
||||||
|
Backlog: (a) de originele weergavenaam herstellen (het label uit de export is niet getest als
|
||||||
|
weergavenaam); (b) de interne `migrated-`-naam hernoemen (vereist directe databasewijziging, niet voor de beta).
|
||||||
|
|
||||||
|
**Backlog uit de pre-beta review (2026-10-07)** — geen belemmering voor 0.9.0-beta of 1.0.0, staat
|
||||||
|
in README "Known limitations":
|
||||||
|
1. Geheugen bij grote instanties (>1 TB, miljoenen bestanden): blob-index, prune-set, browse van een
|
||||||
|
snapshot in de web-UI, `trashExtras` bij restore en het manifest van een user-export staan volledig
|
||||||
|
in het geheugen; streaming/paginering nodig.
|
||||||
|
2. Twee servers op één locatie: catalogus kan breken (generatie overschreven, of anchor >200 generaties
|
||||||
|
achter). Docs-waarschuwing staat er; mogelijk later een echte put-if-absent per backend.
|
||||||
|
3. Sleutelrotatie: een slot verwijderen trekt geen toegang in (master-key blijft gelijk).
|
||||||
|
4. Rollback vóór DR niet detecteerbaar: referentie-generatie en -hash in de kit opnemen of tonen.
|
||||||
|
5. Kit-velden buiten `wrapped_key` (o.a. `delete_delay_days`) zijn niet geauthenticeerd; nu alleen getoond.
|
||||||
|
6. Repository-id in de AD van objecten (formaatwijziging).
|
||||||
|
7. Wachtwoorden van locaties via `--option-file` of omgevingsvariabele i.p.v. de commandoregel.
|
||||||
|
8. Tijdelijke restore-bestanden niet via de Files-API verwijderen (komen nu in de prullenbak).
|
||||||
|
9. DB-restore in foreign-key-volgorde (cascades).
|
||||||
|
10. Packs zonder index (index-upload mislukt) opruimen.
|
||||||
|
11. `backup:restore:full` in onderhoudsmodus en hervatbaar maken; nu: opnieuw draaien met `--force`.
|
||||||
|
12. Config/rename op FTP/SMB-servers die geen rename over een bestaand bestand toestaan.
|
||||||
|
|
||||||
|
Doorgeschoven naar **v1.3.0** (fase 3, niet meer fase 2): in-place volledige restore
|
||||||
|
(eigen restore-modus i.p.v. onderhoudsmodus) en zelfbediening voor gebruikers (eigen bestanden
|
||||||
|
terugzetten vanuit de Files-app).
|
||||||
|
|
||||||
|
**Let op (ontdekt 2026-10-05):** Nextcloud's eigen `info.xsd` staat voor `<version>` alleen
|
||||||
|
`[0-9]+(\.[0-9]+){0,2}` toe — geen `-alpha`/`-beta`/`-rc`-achtervoegsel. De stabiliteitsaanduiding
|
||||||
|
uit deze tabel gaat dus niet in `info.xml` (dat blijft gewoon oplopend, bijv. `0.1.0`), maar in de
|
||||||
|
git-tag (`v0.1.0-alpha.1`), de GitHub/Gitea release (gemarkeerd als pre-release) en de CHANGELOG.
|
||||||
|
|
||||||
## Bijlage: lessen uit nextcloud/backup (test NC34, 2026-10-01)
|
## Bijlage: lessen uit nextcloud/backup (test NC34, 2026-10-01)
|
||||||
|
|
||||||
- Externe programma's (`pg_dump`) en extensies (`pgsql`, `mysqli`) → onbruikbaar op standaard/hosted.
|
- Externe programma's (`pg_dump`) en extensies (`pgsql`, `mysqli`) → onbruikbaar op standaard/hosted.
|
||||||
|
|||||||
@@ -1,8 +1,203 @@
|
|||||||
# NG Backup
|
# NG Backup
|
||||||
|
|
||||||
Encrypted, incremental backups of Nextcloud with restore from the web interface.
|
Encrypted, incremental backups of Nextcloud (database, configuration and data) with restore
|
||||||
Works without shell access: no external programs, no extra PHP extensions beyond sodium.
|
from the web interface. Works without shell access: no external programs, no extra PHP
|
||||||
|
extensions beyond sodium for ng_backup itself. Runs on Nextcloud All-in-One, regular
|
||||||
|
installations and hosted Nextcloud. (Choosing SMB as a storage backend is the one exception:
|
||||||
|
see "Storage backend experiences" below — that requirement comes from files_external, not from
|
||||||
|
ng_backup, and applies to any Nextcloud app that writes to an SMB share.)
|
||||||
|
|
||||||
**Status: phase 0 (prototype). Not usable yet.** See `PLAN.md` for the design.
|
**Status: beta (0.9.5).** Tested thoroughly, usable alongside your current backup, but not yet a
|
||||||
|
drop-in replacement for it. See "Known limitations" below, `PLAN.md` for the design and
|
||||||
|
`docs/HOWTO.md` for a step-by-step guide from setup to a tested restore.
|
||||||
|
|
||||||
License: AGPL-3.0-or-later.
|
License: AGPL-3.0-or-later.
|
||||||
|
|
||||||
|
## How it works
|
||||||
|
|
||||||
|
- Backups go to a **location**: S3 (and S3-compatible storage), SFTP, WebDAV, FTP, Swift or a
|
||||||
|
local path. NG Backup uses the connection code of Nextcloud's External storage app, but never
|
||||||
|
mounts the location: it is only connected while a backup or restore runs, and it never
|
||||||
|
appears in Files, for anyone.
|
||||||
|
- Data is cut into blocks, deduplicated, compressed and encrypted (XChaCha20-Poly1305) before it
|
||||||
|
leaves the server, and streamed straight to the location: no local copy, no double disk space.
|
||||||
|
Unchanged files are not read again; only new data is uploaded.
|
||||||
|
- The database is dumped through Nextcloud's own connection, inside one consistent transaction.
|
||||||
|
- Restores go through Nextcloud, so overwritten files keep their old content as a version and
|
||||||
|
removed files go to the trash.
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:key:init --label="Safe" # first passphrase, deletion delay (default 7 days)
|
||||||
|
occ backup:key:slot:add --label="CTO" # optional second and third passphrase
|
||||||
|
occ backup:key:slot:add --label="Head of IT"
|
||||||
|
occ backup:key:kit --output=/var/tmp/ng_backup-kit.json
|
||||||
|
occ backup:key:confirm
|
||||||
|
occ backup:target:add offsite amazons3 -a amazons3::accesskey -o bucket=... -o key=... -o secret=...
|
||||||
|
occ backup:run
|
||||||
|
```
|
||||||
|
|
||||||
|
Or use **Administration settings → NG Backup**.
|
||||||
|
|
||||||
|
## The backup key, passphrases and the recovery kit
|
||||||
|
|
||||||
|
All backups of an installation are encrypted with one master key.
|
||||||
|
|
||||||
|
- The **server** keeps the master key, encrypted with the instance secret, so scheduled backups
|
||||||
|
and restores work without anyone typing a passphrase. Any administrator can restore from the
|
||||||
|
web interface. Consequence: whoever fully controls the server can read the backups; whoever
|
||||||
|
only gets hold of the backup location cannot.
|
||||||
|
- For a **restore on a new server** (disaster recovery) you need the **recovery kit** and **one
|
||||||
|
passphrase**. The kit contains the master key, encrypted separately with each passphrase.
|
||||||
|
- Up to **three passphrases** ("slots") can open the key, so one person leaving or dying does not
|
||||||
|
lock the company out. A suggested setup:
|
||||||
|
|
||||||
|
| Slot | Kept by |
|
||||||
|
|---|---|
|
||||||
|
| 1 | in the safe (sealed envelope) |
|
||||||
|
| 2 | the CTO |
|
||||||
|
| 3 | the head of IT |
|
||||||
|
|
||||||
|
Every holder keeps a copy of the same recovery kit and only knows their own passphrase. Using
|
||||||
|
a single slot is possible, but then that one passphrase is a single point of failure.
|
||||||
|
- When a holder leaves, give their slot a new passphrase (`occ backup:key:slot:replace`), download
|
||||||
|
the new kit and confirm it. Note: an old kit together with the old passphrase still opens the
|
||||||
|
same master key; to revoke it completely, start new locations with a new key. This is noted on
|
||||||
|
the backlog and will be addressed in a future version (key rotation).
|
||||||
|
- Downloading the kit, and every change to the slots, requires the administrator's password, is
|
||||||
|
written to the audit log and is announced to all administrators.
|
||||||
|
|
||||||
|
## Protection against ransomware and a compromised server
|
||||||
|
|
||||||
|
A backup is only useful if an attacker cannot destroy it. An attacker with administrator access
|
||||||
|
to Nextcloud could try to remove old restore points so that only infected data remains.
|
||||||
|
NG Backup makes this hard and visible, but **software alone can never guarantee it**: the
|
||||||
|
location must refuse deletions itself.
|
||||||
|
|
||||||
|
What NG Backup does:
|
||||||
|
|
||||||
|
- **Deletion delay.** Removed restore points first go to a trash on the location and are only
|
||||||
|
deleted after the delay chosen at installation (default 7 days, cannot be changed afterwards).
|
||||||
|
Within that time any administrator can bring them back (`occ backup:trash:restore`, or in the
|
||||||
|
web interface).
|
||||||
|
- **Notifications to all administrators** when restore points are removed, the retention is
|
||||||
|
changed, append-only is turned off, a location is removed or the recovery kit is downloaded,
|
||||||
|
and an entry in the audit log (admin_audit).
|
||||||
|
- **Append-only locations** (`occ backup:target:append-only NAME on`): NG Backup never deletes
|
||||||
|
anything there; old data is cleaned up by the location's own lifecycle rules.
|
||||||
|
- Removing a location in NG Backup only removes the connection. The backups on the location are
|
||||||
|
not touched and nothing keeps cleaning up in the background.
|
||||||
|
|
||||||
|
What you should do on the location (strongly recommended):
|
||||||
|
|
||||||
|
- **S3: enable Object Lock** on the bucket, in **compliance mode**, with a default retention of
|
||||||
|
for example **90 days**, plus versioning. Then nobody, not even with the server's credentials,
|
||||||
|
can delete or overwrite backups younger than 90 days. Combine with an append-only location in
|
||||||
|
NG Backup and a lifecycle rule that removes old versions after the lock expires.
|
||||||
|
With Object Lock, a deletion by NG Backup (or by an attacker through it) only adds a *delete
|
||||||
|
marker*: the locked data stays and can be brought back from the bucket's object versions until
|
||||||
|
the lock expires. Tested with an S3-compatible server; still to be verified on AWS itself, which
|
||||||
|
requires a checksum on uploads to locked buckets.
|
||||||
|
- Use **credentials without delete rights** for the backup location where the storage supports it.
|
||||||
|
- **SFTP / storage boxes:** enable the provider's own snapshots (e.g. daily, kept 30-90 days) and
|
||||||
|
give the backup user access to its own folder only, without a shell.
|
||||||
|
- Keep at least one location **off-site**, on a different network than the Nextcloud server.
|
||||||
|
|
||||||
|
## Restore
|
||||||
|
|
||||||
|
- **Files and folders** of a user: in the web interface or `occ backup:restore`, into a new folder
|
||||||
|
("Restored <date>"), merged into the original place, or replacing it.
|
||||||
|
- **Raw restore** of any path (for example `config/`) to a local directory: `occ backup:restore ... --to-directory=DIR`.
|
||||||
|
- **Whole server** (disaster recovery onto a fresh installation): `occ backup:restore:full`, see the how-to.
|
||||||
|
- **One user** (account, settings, files, calendars, contacts via user_migration): `occ backup:user:backup` and
|
||||||
|
`occ backup:user:restore`, replacing the account or restoring it as `<uid>-bak`. See the how-to.
|
||||||
|
|
||||||
|
## Storage backend experiences
|
||||||
|
|
||||||
|
Notes from actually running a backend, beyond what's in files_external's own documentation.
|
||||||
|
Contributions welcome via PR — add what you ran into with your own provider.
|
||||||
|
|
||||||
|
- **Hetzner Storage Box, sub-account:** the SFTP backend's `root` option must be `home`, not `/`.
|
||||||
|
A sub-account's SFTP login lands in `/home` (confirmed via `pwd()` after login); writing to the
|
||||||
|
literal chroot root (`/`) fails with a generic `NET_SFTP_STATUS_FAILURE` from the SFTP server,
|
||||||
|
while the same write under `home` (or a path below it) succeeds. This affects any backend option
|
||||||
|
form that lets you set a path relative to the account's root, not just ng_backup. Not yet
|
||||||
|
verified whether this also applies to a Hetzner Storage Box **main** account (only tested with a
|
||||||
|
sub-account so far) — if you've tried a main account, please say so in a PR.
|
||||||
|
|
||||||
|
- **Hetzner Storage Box over WebDAV and SMB (sub-accounts, 2026-10-07):** both work end to end
|
||||||
|
(backup, incremental run, restore of a folder, verify), tested from outside the Hetzner network
|
||||||
|
with about 66 MiB of data. In the Hetzner Console, enable WebDAV or SMB support for the
|
||||||
|
sub-account, and "External reachability" when your server is not in the Hetzner network.
|
||||||
|
WebDAV: host `https://uXXXXX-subN.your-storagebox.de`, and the `root`
|
||||||
|
must be a folder that already exists (or empty): WebDAV does not create missing parent folders,
|
||||||
|
so a root that doesn't exist yet fails `backup:target:add` with `Sabre\HTTP\ClientHttpException:
|
||||||
|
Conflict`. SMB: share = the sub-account name, port 445 must be reachable from your server (it was
|
||||||
|
from a home connection here, but some providers block it), and the `smbclient` PECL extension is
|
||||||
|
required (see the SMB entry below). Both have a noticeable fixed cost per run: an incremental run
|
||||||
|
that uploads nothing took about 20 s over WebDAV and 50 s over SMB. Over WebDAV, Nextcloud gives
|
||||||
|
every request at most 30 s by default, also an upload of a whole 32 MiB pack: on an upload
|
||||||
|
slower than about 9 Mbit/s a backup fails with `cURL error 28: Operation timed out`. Raise the
|
||||||
|
limit with `occ config:system:set davstorage.request_timeout --value=600 --type=integer`.
|
||||||
|
|
||||||
|
- **FTP, server-dependent absolute-path handling:** files_external's FTP backend always builds
|
||||||
|
absolute paths (e.g. `MKD /repo`) for the configured root, never relative ones. `atmoz/sftp`-style
|
||||||
|
`vsftpd` test servers (chrooted) can reject that with a generic "create directory operation
|
||||||
|
failed", even though a plain `ftp_mkdir()` with a *relative* path on the same server succeeds;
|
||||||
|
`pure-ftpd` accepts the absolute form fine. If a working FTP server rejects ng_backup's writes,
|
||||||
|
try a different FTP server implementation before assuming the backend itself is broken.
|
||||||
|
|
||||||
|
- **SMB/CIFS *lists* as available with just the `smbclient` CLI binary, but does not actually
|
||||||
|
work for ng_backup without the `smbclient` PECL extension too.** files_external falls back to a
|
||||||
|
shell-wrapped `smbclient` process when the native extension is missing, which registers the
|
||||||
|
backend in `occ backup:target:backends` fine, but that fallback's stream cannot seek. ng_backup
|
||||||
|
reads blobs back by byte range while backing up, so even the very first (small) backup fails
|
||||||
|
with `Seek failed in packs/...`. Install the extension (`libsmbclient-dev` + `pecl install
|
||||||
|
smbclient` + `docker-php-ext-enable smbclient` on Debian-based images), not just the CLI tool —
|
||||||
|
the backend being listed is not proof it will work here. Once the extension is present it works
|
||||||
|
the same as any other backend; no ng_backup-specific config beyond host/share/user/password.
|
||||||
|
|
||||||
|
- **NFS is not a files_external backend at all.** Mount the NFS export on the server yourself
|
||||||
|
(e.g. via `/etc/fstab` or a systemd `.mount` unit) and point ng_backup's `local` backend at the
|
||||||
|
mount point; ng_backup never mounts or unmounts anything on its own, so an NFS mount dropping
|
||||||
|
mid-run behaves like any other local-path failure (the run resumes). See PLAN.md 3.5 for why NFS
|
||||||
|
can't be driven from PHP directly.
|
||||||
|
|
||||||
|
## Known limitations
|
||||||
|
|
||||||
|
On the backlog; none of these block the beta or 1.0.0, but know them before you rely on it:
|
||||||
|
|
||||||
|
- **Very large instances** (well over 1 TB, millions of files): the blob index, cleanup, browsing a
|
||||||
|
snapshot in the web interface and per-user export manifests are held in memory. Expect high
|
||||||
|
memory use there; reports from such instances are very welcome.
|
||||||
|
- **One location, one server.** Never let two Nextcloud servers write to the same location (for
|
||||||
|
example a disaster recovery test server pointed at the production location): they can break each
|
||||||
|
other's catalog, and the location is then refused as possibly tampered with. Test a disaster
|
||||||
|
recovery against a copy of the location, or only read from it.
|
||||||
|
- **Removing a passphrase slot does not revoke access.** The master key stays the same, so someone
|
||||||
|
with an old recovery kit and that slot's passphrase can still decrypt the location if they can
|
||||||
|
read it. Key rotation is not available yet.
|
||||||
|
- **Rollback before a disaster recovery is not detected.** A fresh server has no record of the
|
||||||
|
newest catalog generation, so a location rolled back to an older state looks valid there.
|
||||||
|
- Location passwords passed with `-o password=...` are visible in the process list and shell
|
||||||
|
history while the command runs.
|
||||||
|
- Temporary files of a restore into a user's files end up in that user's trash.
|
||||||
|
- Restoring the database does not order tables by foreign keys; apps with cascading foreign keys
|
||||||
|
could lose rows (Nextcloud core does not use them).
|
||||||
|
- A pack whose index upload failed stays on the location until removed by hand.
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
|
||||||
|
| Command | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `backup:key:init`, `backup:key:kit`, `backup:key:confirm` | create the key, recovery kit, confirmation |
|
||||||
|
| `backup:key:slots`, `backup:key:slot:add/replace/remove` | passphrase slots |
|
||||||
|
| `backup:target:backends/add/edit/list/remove/append-only` | backup locations |
|
||||||
|
| `backup:run`, `backup:status`, `backup:list` | back up, status, restore points |
|
||||||
|
| `backup:browse`, `backup:restore` | browse and restore |
|
||||||
|
| `backup:retention`, `backup:prune`, `backup:trash`, `backup:trash:restore` | retention, cleanup, trash |
|
||||||
|
| `backup:verify` | checksum audit of a restore point (`--deep` downloads and decrypts everything) |
|
||||||
|
| `backup:restore:full` | whole-server restore onto a fresh installation (after `backup:key:import`) |
|
||||||
|
| `backup:key:import` | import a recovery kit on a fresh installation |
|
||||||
|
| `backup:user:backup`, `backup:user:restore` | one user via user_migration (asks before installing it) |
|
||||||
|
|||||||
+36
-3
@@ -4,13 +4,25 @@
|
|||||||
<id>ng_backup</id>
|
<id>ng_backup</id>
|
||||||
<name>NG Backup</name>
|
<name>NG Backup</name>
|
||||||
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
||||||
<description><![CDATA[Work in progress, not ready for use.]]></description>
|
<description><![CDATA[Beta: scheduled, encrypted, incremental backups of the database, config and data
|
||||||
<version>0.0.4</version>
|
directory to S3, SFTP, WebDAV, FTP, SMB or a local path, with restore of files and folders, of
|
||||||
|
single users (via user_migration) and of the whole server onto a fresh installation. Tested
|
||||||
|
thoroughly and usable alongside your current backup, but not yet a drop-in replacement for it.
|
||||||
|
See the README for known limitations and the recovery kit before relying on it.]]></description>
|
||||||
|
<version>0.9.5</version>
|
||||||
<licence>agpl</licence>
|
<licence>agpl</licence>
|
||||||
<author>Dillard Blom</author>
|
<author mail="dillard.blom@ensembia.com">Dillard Blom</author>
|
||||||
<namespace>NgBackup</namespace>
|
<namespace>NgBackup</namespace>
|
||||||
|
<documentation>
|
||||||
|
<admin>https://github.com/dillardblom/ng_backup/blob/main/docs/HOWTO.md</admin>
|
||||||
|
</documentation>
|
||||||
<category>tools</category>
|
<category>tools</category>
|
||||||
|
<website>https://github.com/dillardblom/ng_backup</website>
|
||||||
<bugs>https://github.com/dillardblom/ng_backup/issues</bugs>
|
<bugs>https://github.com/dillardblom/ng_backup/issues</bugs>
|
||||||
|
<repository type="git">https://github.com/dillardblom/ng_backup</repository>
|
||||||
|
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/1-backups.png</screenshot>
|
||||||
|
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/2-restore.png</screenshot>
|
||||||
|
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/3-settings.png</screenshot>
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<php min-version="8.1"/>
|
<php min-version="8.1"/>
|
||||||
<lib>sodium</lib>
|
<lib>sodium</lib>
|
||||||
@@ -25,6 +37,7 @@
|
|||||||
<command>OCA\NgBackup\Command\KeyConfirm</command>
|
<command>OCA\NgBackup\Command\KeyConfirm</command>
|
||||||
<command>OCA\NgBackup\Command\TargetBackends</command>
|
<command>OCA\NgBackup\Command\TargetBackends</command>
|
||||||
<command>OCA\NgBackup\Command\TargetAdd</command>
|
<command>OCA\NgBackup\Command\TargetAdd</command>
|
||||||
|
<command>OCA\NgBackup\Command\TargetEdit</command>
|
||||||
<command>OCA\NgBackup\Command\TargetList</command>
|
<command>OCA\NgBackup\Command\TargetList</command>
|
||||||
<command>OCA\NgBackup\Command\TargetRemove</command>
|
<command>OCA\NgBackup\Command\TargetRemove</command>
|
||||||
<command>OCA\NgBackup\Command\RunBackup</command>
|
<command>OCA\NgBackup\Command\RunBackup</command>
|
||||||
@@ -32,5 +45,25 @@
|
|||||||
<command>OCA\NgBackup\Command\ListSnapshots</command>
|
<command>OCA\NgBackup\Command\ListSnapshots</command>
|
||||||
<command>OCA\NgBackup\Command\Browse</command>
|
<command>OCA\NgBackup\Command\Browse</command>
|
||||||
<command>OCA\NgBackup\Command\Restore</command>
|
<command>OCA\NgBackup\Command\Restore</command>
|
||||||
|
<command>OCA\NgBackup\Command\Prune</command>
|
||||||
|
<command>OCA\NgBackup\Command\Retention</command>
|
||||||
|
<command>OCA\NgBackup\Command\TargetAppendOnly</command>
|
||||||
|
<command>OCA\NgBackup\Command\TargetTrustCurrent</command>
|
||||||
|
<command>OCA\NgBackup\Command\TargetLimit</command>
|
||||||
|
<command>OCA\NgBackup\Command\Trash</command>
|
||||||
|
<command>OCA\NgBackup\Command\TrashRestore</command>
|
||||||
|
<command>OCA\NgBackup\Command\KeySlots</command>
|
||||||
|
<command>OCA\NgBackup\Command\KeySlotAdd</command>
|
||||||
|
<command>OCA\NgBackup\Command\KeySlotReplace</command>
|
||||||
|
<command>OCA\NgBackup\Command\KeySlotRemove</command>
|
||||||
|
<command>OCA\NgBackup\Command\BackupUser</command>
|
||||||
|
<command>OCA\NgBackup\Command\RestoreUser</command>
|
||||||
|
<command>OCA\NgBackup\Command\KeyImport</command>
|
||||||
|
<command>OCA\NgBackup\Command\RestoreFull</command>
|
||||||
|
<command>OCA\NgBackup\Command\Verify</command>
|
||||||
</commands>
|
</commands>
|
||||||
|
<settings>
|
||||||
|
<admin>OCA\NgBackup\Settings\Admin</admin>
|
||||||
|
<admin-section>OCA\NgBackup\Settings\AdminSection</admin-section>
|
||||||
|
</settings>
|
||||||
</info>
|
</info>
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
return [
|
||||||
|
'routes' => [
|
||||||
|
['name' => 'AdminApi#status', 'url' => '/api/status', 'verb' => 'GET'],
|
||||||
|
['name' => 'AdminApi#initKey', 'url' => '/api/key', 'verb' => 'POST'],
|
||||||
|
['name' => 'AdminApi#downloadKit', 'url' => '/api/key/kit', 'verb' => 'GET'],
|
||||||
|
['name' => 'AdminApi#confirmKit', 'url' => '/api/key/confirm', 'verb' => 'POST'],
|
||||||
|
['name' => 'AdminApi#addSlot', 'url' => '/api/key/slots', 'verb' => 'POST'],
|
||||||
|
['name' => 'AdminApi#replaceSlot', 'url' => '/api/key/slots/{slot}', 'verb' => 'PUT'],
|
||||||
|
['name' => 'AdminApi#removeSlot', 'url' => '/api/key/slots/{slot}', 'verb' => 'DELETE'],
|
||||||
|
['name' => 'AdminApi#trash', 'url' => '/api/targets/{targetId}/trash', 'verb' => 'GET'],
|
||||||
|
['name' => 'AdminApi#untrash', 'url' => '/api/targets/{targetId}/trash/{snapshotId}', 'verb' => 'POST'],
|
||||||
|
['name' => 'AdminApi#backends', 'url' => '/api/backends', 'verb' => 'GET'],
|
||||||
|
['name' => 'AdminApi#addTarget', 'url' => '/api/targets', 'verb' => 'POST'],
|
||||||
|
['name' => 'AdminApi#targetSettings', 'url' => '/api/targets/{id}/settings', 'verb' => 'GET'],
|
||||||
|
['name' => 'AdminApi#updateTarget', 'url' => '/api/targets/{id}', 'verb' => 'PUT'],
|
||||||
|
['name' => 'AdminApi#removeTarget', 'url' => '/api/targets/{id}', 'verb' => 'DELETE'],
|
||||||
|
['name' => 'AdminApi#testTarget', 'url' => '/api/targets/{id}/test', 'verb' => 'POST'],
|
||||||
|
['name' => 'AdminApi#setTargetLimit', 'url' => '/api/targets/{id}/limit', 'verb' => 'PUT'],
|
||||||
|
['name' => 'AdminApi#startBackup', 'url' => '/api/backups', 'verb' => 'POST'],
|
||||||
|
['name' => 'AdminApi#setSchedule', 'url' => '/api/schedule', 'verb' => 'PUT'],
|
||||||
|
['name' => 'AdminApi#snapshots', 'url' => '/api/targets/{targetId}/snapshots', 'verb' => 'GET'],
|
||||||
|
['name' => 'AdminApi#browse', 'url' => '/api/targets/{targetId}/snapshots/{snapshotId}/browse', 'verb' => 'GET'],
|
||||||
|
['name' => 'AdminApi#startRestore', 'url' => '/api/restores', 'verb' => 'POST'],
|
||||||
|
],
|
||||||
|
];
|
||||||
+166
@@ -0,0 +1,166 @@
|
|||||||
|
# NG Backup how-to
|
||||||
|
|
||||||
|
Step by step: from first install to a tested restore. All commands run as the web user
|
||||||
|
(`sudo -u www-data php occ ...`, or `docker exec -u www-data <container> php occ ...` in
|
||||||
|
All-in-One and Docker setups). Replace names in `<angle brackets>` with your own.
|
||||||
|
|
||||||
|
## 1. Create the key and the recovery kit
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:key:init --label="Main" --delete-delay-days=7
|
||||||
|
occ backup:key:kit --output=/var/tmp/ng_backup-kit.json
|
||||||
|
occ backup:key:confirm -u <your-admin-uid> --code=<code shown> --confirm="Yes, I confirm"
|
||||||
|
```
|
||||||
|
|
||||||
|
- The passphrase protects the backup. Keep it somewhere other than this server.
|
||||||
|
- Optional: extra passphrases for other people with `occ backup:key:slot:add --label="..."`.
|
||||||
|
Any one passphrase is enough to restore. Removing a slot later does not revoke access for someone
|
||||||
|
who kept an old kit and that passphrase (see "Known limitations" in the README).
|
||||||
|
- The recovery kit (`/var/tmp/ng_backup-kit.json` above) is also required for disaster recovery. The web
|
||||||
|
user writes it there; then move it off the server as root, next to the passphrase, and delete the
|
||||||
|
copy in `/var/tmp`. Without the kit and a passphrase, the backup cannot be read.
|
||||||
|
|
||||||
|
## 2. Add a backup location
|
||||||
|
|
||||||
|
NG Backup uses Nextcloud's External storage backends. Check what your server supports first:
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:target:backends
|
||||||
|
```
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:target:add offsite amazons3 -a amazons3::accesskey -o bucket=<bucket> -o key=<key> -o secret=<secret>
|
||||||
|
occ backup:target:add nas sftp -a password::password -o host=<host> -o user=<user> -o password=<pw> -o root=home
|
||||||
|
occ backup:target:add local local -a null::null -o datadir=/mnt/backup
|
||||||
|
```
|
||||||
|
|
||||||
|
- **SFTP on a Hetzner Storage Box sub-account:** use `-o root=home`, not `/` (see README).
|
||||||
|
- **FTP:** use a server that accepts absolute paths (for example `pure-ftpd`).
|
||||||
|
- **SMB:** requires the native `smbclient` PHP extension. The CLI binary alone is not enough (see README).
|
||||||
|
`backup:target:add` refuses SMB when the extension is missing.
|
||||||
|
- **NFS:** not a backend of its own. Mount the export on the server and use the `local` backend
|
||||||
|
with the mount point as `datadir`.
|
||||||
|
|
||||||
|
Verify the location before relying on it:
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:target:list
|
||||||
|
occ backup:status
|
||||||
|
```
|
||||||
|
|
||||||
|
Changed password, new host name, or moved the repository? Edit the location instead of adding a
|
||||||
|
new one, so its history stays (in the web interface: **Edit** next to the location). The new
|
||||||
|
settings must lead to the same repository; options you do not pass keep their value:
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:target:edit nas -o password=<new pw>
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. Run the first backup
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:run offsite -l first
|
||||||
|
occ backup:list
|
||||||
|
```
|
||||||
|
|
||||||
|
Later runs upload only changed data. An unchanged run uploads (almost) nothing; the output shows
|
||||||
|
the number of uploaded bytes, so check that value when you test incremental behaviour.
|
||||||
|
|
||||||
|
Schedule `occ backup:run` from system cron (or use the Administration settings, if you prefer).
|
||||||
|
|
||||||
|
## 4. Check the backup regularly
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:verify offsite <snapshot> # shallow: all blocks are present
|
||||||
|
occ backup:verify offsite <snapshot> --deep # also downloads and decrypts every block
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the deep check at least once after setting up a location, and after changing it.
|
||||||
|
|
||||||
|
## 5. Restore files and folders
|
||||||
|
|
||||||
|
In the web interface, or with occ:
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:browse offsite <snapshot>
|
||||||
|
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=new-folder
|
||||||
|
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=merge
|
||||||
|
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=replace
|
||||||
|
occ backup:restore offsite <snapshot> config --to-directory=/var/tmp/restore-config
|
||||||
|
```
|
||||||
|
|
||||||
|
- `new-folder` (default): puts the files in "Restored <date>" next to the original. Safest choice.
|
||||||
|
- `merge`: adds missing files, keeps current ones.
|
||||||
|
- `replace`: overwrites the folder. Replaced files keep their old content as a version, and removed
|
||||||
|
files go to the trash.
|
||||||
|
- `--to-directory`: writes raw files to a local directory (for example `config/`).
|
||||||
|
|
||||||
|
## 6. Back up and restore one user
|
||||||
|
|
||||||
|
Needs the `user_migration` app. NG Backup asks before installing it:
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:user:backup offsite <uid>
|
||||||
|
occ backup:user:restore offsite users/<uid>/<exportId> # replace (default)
|
||||||
|
occ backup:user:restore offsite users/<uid>/<exportId> --mode=as-backup
|
||||||
|
```
|
||||||
|
|
||||||
|
- If `user_migration` is missing, the command asks to install and enable it. For non-interactive
|
||||||
|
use, add `--install-user-migration` to approve that explicitly. The command stops after the
|
||||||
|
installation; run it once more.
|
||||||
|
- `replace` first makes a safety export of the current account, then recreates the user under the
|
||||||
|
same id. When the account exists, the command asks first; in scripts, add `--force`. Shares to
|
||||||
|
others and data of apps without a user_migration migrator are not restored, so prefer
|
||||||
|
`as-backup` when in doubt: it restores into `<uid>-bak` and leaves the original untouched.
|
||||||
|
- A restore onto a server where the user does not exist works. Calendars and address books get an
|
||||||
|
internal name with the prefix `migrated-` (for example `migrated-personal`). That prefix comes from
|
||||||
|
user_migration and cannot be turned off. The user can change the display name of these calendars and
|
||||||
|
address books in the app as usual; the internal name stays as it is.
|
||||||
|
- **Settings:** on restore, `user_migration` imports only the app settings on its own allowlist
|
||||||
|
(for example calendar view and reminder settings). Other app settings of the user are skipped.
|
||||||
|
NG Backup reuses `user_migration`'s importer, so this list is not extended.
|
||||||
|
- Keep the last few exports per user: `occ backup:retention --user-last=3`.
|
||||||
|
|
||||||
|
## 7. Disaster recovery: restore the whole server
|
||||||
|
|
||||||
|
Use a fresh Nextcloud installation of the same major version, with NG Backup installed and
|
||||||
|
the same location available. For a test, use a **copy** of the location: two servers writing to one
|
||||||
|
location break each other's catalog.
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:key:import /path/ng_backup-kit.json --passphrase-file=/path/passphrase
|
||||||
|
occ backup:target:add offsite <backend> ... # same details as on the original server
|
||||||
|
occ backup:list offsite # the snapshots must be listed
|
||||||
|
occ backup:restore:full offsite <snapshot>
|
||||||
|
```
|
||||||
|
|
||||||
|
The command refuses on an installation that already has more than one user, and asks before it
|
||||||
|
starts; `--force` skips both (also needed to rerun it after it stopped halfway).
|
||||||
|
|
||||||
|
Afterwards, check that users, files and the database are complete, and run a backup of the restored
|
||||||
|
server. The restored installation keeps its own app configuration and secret.
|
||||||
|
|
||||||
|
## 8. Retention
|
||||||
|
|
||||||
|
```
|
||||||
|
occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12
|
||||||
|
occ backup:prune offsite --dry-run # show first
|
||||||
|
occ backup:prune offsite # apply the policy; forgotten snapshots go to the trash first
|
||||||
|
occ backup:trash # list the trash
|
||||||
|
occ backup:trash:restore offsite <id>
|
||||||
|
```
|
||||||
|
|
||||||
|
Snapshots are kept according to the policy, and their data is deleted only after the deletion delay
|
||||||
|
(set with `backup:key:init --delete-delay-days`). `occ backup:prune offsite --dry-run` shows what would be
|
||||||
|
removed without changing anything.
|
||||||
|
|
||||||
|
## 9. Checklist before you rely on it
|
||||||
|
|
||||||
|
- [ ] Key created, passphrase stored off-server, recovery kit stored off-server
|
||||||
|
- [ ] Location added; `backup:status` shows it reachable
|
||||||
|
- [ ] First backup done and listed in `backup:list`
|
||||||
|
- [ ] `backup:verify --deep` succeeds
|
||||||
|
- [ ] One file restore tested (`new-folder`)
|
||||||
|
- [ ] Disaster recovery tested on a second, fresh installation (or in a test environment)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24"><path fill="#000" d="M12 2 4 5v6c0 5 3.4 9.7 8 11 4.6-1.3 8-6 8-11V5l-8-3Zm0 5a3 3 0 0 1 3 3v1h1v6H8v-6h1v-1a3 3 0 0 1 3-3Zm0 1.6c-.8 0-1.4.6-1.4 1.4v1h2.8v-1c0-.8-.6-1.4-1.4-1.4Z"/></svg>
|
||||||
|
After Width: | Height: | Size: 272 B |
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24"><path fill="#fff" d="M12 2 4 5v6c0 5 3.4 9.7 8 11 4.6-1.3 8-6 8-11V5l-8-3Zm0 5a3 3 0 0 1 3 3v1h1v6H8v-6h1v-1a3 3 0 0 1 3-3Zm0 1.6c-.8 0-1.4.6-1.4 1.4v1h2.8v-1c0-.8-.6-1.4-1.4-1.4Z"/></svg>
|
||||||
|
After Width: | Height: | Size: 272 B |
@@ -12,7 +12,7 @@ use OCP\AppFramework\Bootstrap\IBootContext;
|
|||||||
use OCP\AppFramework\Bootstrap\IBootstrap;
|
use OCP\AppFramework\Bootstrap\IBootstrap;
|
||||||
use OCP\AppFramework\Bootstrap\IRegistrationContext;
|
use OCP\AppFramework\Bootstrap\IRegistrationContext;
|
||||||
|
|
||||||
class Application extends App implements IBootstrap {
|
final class Application extends App implements IBootstrap {
|
||||||
public const APP_ID = 'ng_backup';
|
public const APP_ID = 'ng_backup';
|
||||||
|
|
||||||
public function __construct() {
|
public function __construct() {
|
||||||
@@ -20,6 +20,8 @@ class Application extends App implements IBootstrap {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function register(IRegistrationContext $context): void {
|
public function register(IRegistrationContext $context): void {
|
||||||
|
$context->registerNotifierService(\OCA\NgBackup\Notification\Notifier::class);
|
||||||
|
$context->registerSetupCheck(\OCA\NgBackup\SetupCheck\BackupStatusCheck::class);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function boot(IBootContext $context): void {
|
public function boot(IBootContext $context): void {
|
||||||
|
|||||||
@@ -7,5 +7,5 @@ declare(strict_types=1);
|
|||||||
|
|
||||||
namespace OCA\NgBackup\Backend;
|
namespace OCA\NgBackup\Backend;
|
||||||
|
|
||||||
class BackendException extends \RuntimeException {
|
final class BackendException extends \RuntimeException {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,16 +26,32 @@ final class ExternalStorageFactory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return list<array{id:string, name:string, auth:list<string>, parameters:array<string, mixed>}>
|
* @return list<array{id:string, name:string, auth:list<array{id:string, name:string, parameters:array}>, parameters:array<string, mixed>}>
|
||||||
* available backends, to build the settings form from
|
* available backends, to build the settings form from
|
||||||
*/
|
*/
|
||||||
public function describeBackends(): array {
|
public function describeBackends(): array {
|
||||||
$result = [];
|
$result = [];
|
||||||
foreach ($this->backends->getAvailableBackends() as $backend) {
|
foreach ($this->backends->getAvailableBackends() as $backend) {
|
||||||
|
$auth = [];
|
||||||
|
foreach ($this->backends->getAuthMechanismsByScheme(array_keys($backend->getAuthSchemes())) as $mech) {
|
||||||
|
// Mechanisms that need the logged-in user's session cannot work for unattended backups.
|
||||||
|
if (in_array($mech->getScheme(), ['builtin', 'null'], true) && $mech->getIdentifier() !== 'null::null') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (str_starts_with($mech->getIdentifier(), 'password::session') || str_starts_with($mech->getIdentifier(), 'password::logincredentials')
|
||||||
|
|| str_starts_with($mech->getIdentifier(), 'password::userprovided') || str_starts_with($mech->getIdentifier(), 'password::global')) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$auth[] = [
|
||||||
|
'id' => $mech->getIdentifier(),
|
||||||
|
'name' => $mech->getText(),
|
||||||
|
'parameters' => array_map(fn ($p) => $p->jsonSerialize(), $mech->getParameters()),
|
||||||
|
];
|
||||||
|
}
|
||||||
$result[] = [
|
$result[] = [
|
||||||
'id' => $backend->getIdentifier(),
|
'id' => $backend->getIdentifier(),
|
||||||
'name' => $backend->getText(),
|
'name' => $backend->getText(),
|
||||||
'auth' => array_keys($backend->getAuthSchemes()),
|
'auth' => $auth,
|
||||||
'parameters' => array_map(fn ($p) => $p->jsonSerialize(), $backend->getParameters()),
|
'parameters' => array_map(fn ($p) => $p->jsonSerialize(), $backend->getParameters()),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Backend;
|
||||||
|
|
||||||
|
/** IBackend::getMany() for a backend where reading files one after the other costs nothing extra. */
|
||||||
|
trait GetsOneByOne {
|
||||||
|
/**
|
||||||
|
* @param list<string> $paths
|
||||||
|
* @return \Generator<string, string>
|
||||||
|
*/
|
||||||
|
public function getMany(array $paths): \Generator {
|
||||||
|
foreach ($paths as $path) {
|
||||||
|
$fh = $this->get($path);
|
||||||
|
try {
|
||||||
|
$data = stream_get_contents($fh);
|
||||||
|
} finally {
|
||||||
|
fclose($fh);
|
||||||
|
}
|
||||||
|
if ($data === false) {
|
||||||
|
throw new BackendException('Read failed for ' . $path);
|
||||||
|
}
|
||||||
|
yield $path => $data;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -22,13 +22,40 @@ interface IBackend {
|
|||||||
/** @return resource readable stream of the whole object */
|
/** @return resource readable stream of the whole object */
|
||||||
public function get(string $path);
|
public function get(string $path);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Many small objects whole, in the order given (the repository's index files). A backend
|
||||||
|
* where every request costs a round trip can fetch them several at a time.
|
||||||
|
*
|
||||||
|
* @param list<string> $paths
|
||||||
|
* @return \Generator<string, string> contents by path
|
||||||
|
* @throws BackendException for an object that cannot be read
|
||||||
|
*/
|
||||||
|
public function getMany(array $paths): \Generator;
|
||||||
|
|
||||||
/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
|
/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
|
||||||
public function getRange(string $path, int $offset, int $length): string;
|
public function getRange(string $path, int $offset, int $length): string;
|
||||||
|
|
||||||
public function exists(string $path): bool;
|
public function exists(string $path): bool;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Move an object within the same location (used for the trash). Backends must use a native
|
||||||
|
* rename, not a read+write copy: some storages (e.g. SFTP) cannot reliably read and write on
|
||||||
|
* the same connection at once, since the read side is a lazily-pulled stream.
|
||||||
|
*/
|
||||||
|
public function move(string $from, string $to): void;
|
||||||
|
|
||||||
/** @return list<string> object paths below $prefix (recursive) */
|
/** @return list<string> object paths below $prefix (recursive) */
|
||||||
public function list(string $prefix): array;
|
public function list(string $prefix): array;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Object paths directly in $dir, for a folder that only holds objects (index/, snapshots/,
|
||||||
|
* catalog/, trash/info/). One listing, without checking the type of every entry: on a remote
|
||||||
|
* storage list() costs a round trip per entry, which for index/ (one file per pack) grows
|
||||||
|
* with the size of the repository.
|
||||||
|
*
|
||||||
|
* @return list<string>
|
||||||
|
*/
|
||||||
|
public function listFiles(string $dir): array;
|
||||||
|
|
||||||
public function delete(string $path): void;
|
public function delete(string $path): void;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ declare(strict_types=1);
|
|||||||
namespace OCA\NgBackup\Backend;
|
namespace OCA\NgBackup\Backend;
|
||||||
|
|
||||||
final class LocalBackend implements IBackend {
|
final class LocalBackend implements IBackend {
|
||||||
|
use GetsOneByOne;
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private string $root,
|
private string $root,
|
||||||
) {
|
) {
|
||||||
@@ -33,11 +35,16 @@ final class LocalBackend implements IBackend {
|
|||||||
if ($out === false) {
|
if ($out === false) {
|
||||||
throw new BackendException('Cannot write ' . $path);
|
throw new BackendException('Cannot write ' . $path);
|
||||||
}
|
}
|
||||||
|
$expected = StorageBackend::remaining($stream);
|
||||||
if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) {
|
if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) {
|
||||||
throw new BackendException('Write failed for ' . $path);
|
throw new BackendException('Write failed for ' . $path);
|
||||||
}
|
}
|
||||||
fclose($out);
|
fclose($out);
|
||||||
$out = false;
|
$out = false;
|
||||||
|
clearstatcache(true, $tmp);
|
||||||
|
if ($expected !== null && filesize($tmp) !== $expected) {
|
||||||
|
throw new BackendException('Incomplete write for ' . $path);
|
||||||
|
}
|
||||||
if (!rename($tmp, $target)) {
|
if (!rename($tmp, $target)) {
|
||||||
throw new BackendException('Cannot finalise ' . $path);
|
throw new BackendException('Cannot finalise ' . $path);
|
||||||
}
|
}
|
||||||
@@ -94,7 +101,7 @@ final class LocalBackend implements IBackend {
|
|||||||
$result = [];
|
$result = [];
|
||||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS));
|
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS));
|
||||||
foreach ($it as $file) {
|
foreach ($it as $file) {
|
||||||
if ($file->isFile() && !str_contains($file->getFilename(), '.part-')) {
|
if ($file->isFile() && !StorageBackend::isTemporary($file->getFilename())) {
|
||||||
$result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/');
|
$result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -102,10 +109,40 @@ final class LocalBackend implements IBackend {
|
|||||||
return $result;
|
return $result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function listFiles(string $dir): array {
|
||||||
|
$base = $this->abs($dir);
|
||||||
|
if (!is_dir($base)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
$result = [];
|
||||||
|
$names = scandir($base);
|
||||||
|
if ($names === false) {
|
||||||
|
throw new BackendException('Cannot list ' . $dir);
|
||||||
|
}
|
||||||
|
foreach ($names as $name) {
|
||||||
|
if ($name !== '.' && $name !== '..' && !StorageBackend::isTemporary($name)) {
|
||||||
|
$result[] = trim($dir, '/') . '/' . $name;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort($result);
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
public function delete(string $path): void {
|
public function delete(string $path): void {
|
||||||
@unlink($this->abs($path));
|
@unlink($this->abs($path));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function move(string $from, string $to): void {
|
||||||
|
$target = $this->abs($to);
|
||||||
|
$dir = dirname($target);
|
||||||
|
if (!is_dir($dir) && !mkdir($dir, 0700, true) && !is_dir($dir)) {
|
||||||
|
throw new BackendException('Cannot create ' . $dir);
|
||||||
|
}
|
||||||
|
if (!rename($this->abs($from), $target)) {
|
||||||
|
throw new BackendException("Cannot move $from to $to");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private function abs(string $path): string {
|
private function abs(string $path): string {
|
||||||
if (str_contains($path, '..') || str_starts_with($path, '/')) {
|
if (str_contains($path, '..') || str_starts_with($path, '/')) {
|
||||||
throw new BackendException('Invalid path: ' . $path);
|
throw new BackendException('Invalid path: ' . $path);
|
||||||
|
|||||||
@@ -0,0 +1,628 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Backend;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read stream for files_external's SFTP storage that keeps several READ requests in flight.
|
||||||
|
*
|
||||||
|
* files_external's own read stream asks for the next 256 KiB only after the previous one has
|
||||||
|
* arrived, so every request costs a full round trip: about 5 MB/s at 47 ms to a Hetzner Storage
|
||||||
|
* Box, whatever the line speed. With 8 requests in flight the same connection reached 15 MB/s.
|
||||||
|
* phpseclib's get() pipelines too, but only into memory or a local file, and only from offset 0.
|
||||||
|
*
|
||||||
|
* Talks to the storage's phpseclib connection through the same internal methods files_external's
|
||||||
|
* read stream uses (_send_sftp_packet, _get_sftp_packet, _realpath); open() returns null when the
|
||||||
|
* connection does not offer them (phpseclib 3 does not), and StorageBackend then falls back to the
|
||||||
|
* storage's own stream. A restore must rather be slow than fail: when a read fails halfway, the
|
||||||
|
* stream goes on from the same position with the storage's own stream. After MAX_FAILURES streams
|
||||||
|
* in a row failed, open() returns null for the rest of the process; one that works resets the count.
|
||||||
|
* Like that stream, it owns the connection while open: no other request may go over the same
|
||||||
|
* connection until it is closed (PackReaderGuard takes care of that for the repository).
|
||||||
|
*/
|
||||||
|
final class SftpReadStream {
|
||||||
|
private const PROTOCOL = 'ngbsftpread';
|
||||||
|
/** Requests in flight once a stream reads sequentially; it starts at one and doubles. */
|
||||||
|
public const DEPTH = 8;
|
||||||
|
/** Files read at once by readMany(): each holds a handle on the server while its batch runs. */
|
||||||
|
public const BATCH = 32;
|
||||||
|
/** Larger files are left to the usual read by readMany(), which holds every file in memory. */
|
||||||
|
private const MANY_MAX_BYTES = 16 * 1048576;
|
||||||
|
/** OpenSSH and the Hetzner Storage Box both answer at most 255 KiB per READ. */
|
||||||
|
private const CHUNK = 261120;
|
||||||
|
/** Every SFTP server must handle packets of this size (draft-ietf-secsh-filexfer-02, section 3). */
|
||||||
|
private const MIN_CHUNK = 32768;
|
||||||
|
|
||||||
|
private const FXP_OPEN = 3;
|
||||||
|
private const FXP_CLOSE = 4;
|
||||||
|
private const FXP_READ = 5;
|
||||||
|
private const FXP_STATUS = 101;
|
||||||
|
private const FXP_HANDLE = 102;
|
||||||
|
private const FXP_DATA = 103;
|
||||||
|
private const FXF_READ = 1;
|
||||||
|
private const FX_OK = 0;
|
||||||
|
private const FX_EOF = 1;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* READ size per connection, lowered when a server answers less than asked and then goes on, so
|
||||||
|
* later streams over that connection ask for no more than it gives.
|
||||||
|
*
|
||||||
|
* @var \WeakMap<object, int>|null
|
||||||
|
*/
|
||||||
|
private static ?\WeakMap $chunks = null;
|
||||||
|
private static int $requestId = 0x4e470000;
|
||||||
|
/**
|
||||||
|
* Connections whose server did not confirm closing a batch's files: readMany() is not used on
|
||||||
|
* them again, so handles cannot pile up.
|
||||||
|
*
|
||||||
|
* @var \WeakMap<object, true>|null
|
||||||
|
*/
|
||||||
|
private static ?\WeakMap $noBatches = null;
|
||||||
|
/** Streams in a row that failed; at MAX_FAILURES only the storage's own stream is used. */
|
||||||
|
private static int $failures = 0;
|
||||||
|
public const MAX_FAILURES = 3;
|
||||||
|
/** Set by stream_open() when the connection failed rather than the file. */
|
||||||
|
private static bool $openBroke = false;
|
||||||
|
/** Why the last stream_open() failed on the connection. */
|
||||||
|
private static string $openCause = '';
|
||||||
|
|
||||||
|
/** @var resource|null set by PHP for stream wrappers */
|
||||||
|
public $context;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The phpseclib connection. PHP creates stream wrappers without constructor arguments; it is
|
||||||
|
* set in stream_open(), and no other method is called when that fails.
|
||||||
|
*
|
||||||
|
* @psalm-suppress PropertyNotSetInConstructor
|
||||||
|
*/
|
||||||
|
private object $sftp;
|
||||||
|
/** The storage and the path on it, to fall back to the storage's own stream. */
|
||||||
|
private ?object $storage = null;
|
||||||
|
private string $file = '';
|
||||||
|
/** @var resource|null the storage's own stream, once this one has failed */
|
||||||
|
private $fallback = null;
|
||||||
|
private string $handle = '';
|
||||||
|
/** Position of the reader: the first byte of $buffer. */
|
||||||
|
private int $pos = 0;
|
||||||
|
/** Offset of the next READ to send. */
|
||||||
|
private int $next = 0;
|
||||||
|
private string $buffer = '';
|
||||||
|
/** @var list<array{int, int, int}> request id, offset, length, in the order sent */
|
||||||
|
private array $inFlight = [];
|
||||||
|
private int $depth = 1;
|
||||||
|
private bool $eof = false;
|
||||||
|
/** This stream gave up on its own requests. */
|
||||||
|
private bool $failed = false;
|
||||||
|
/** The connection is in an unknown state: nothing more is sent over it or read from it. */
|
||||||
|
private bool $broken = false;
|
||||||
|
/** Length of the last short answer, until the next answer tells whether it was the end of the file. */
|
||||||
|
private ?int $shortAnswer = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param object $storage files_external's SFTP storage (getConnection(), getRoot())
|
||||||
|
* @return resource|false|null the stream, false when the file cannot be opened, null when the
|
||||||
|
* connection does not offer what this stream needs
|
||||||
|
* @throws BackendException when the connection failed after the OPEN request went out: the
|
||||||
|
* storage's own stream must not be tried on it, as it would take a
|
||||||
|
* late answer to that request for its own
|
||||||
|
*/
|
||||||
|
public static function open(object $storage, string $path) {
|
||||||
|
if (self::$failures >= self::MAX_FAILURES) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$sftp = self::connection($storage);
|
||||||
|
if ($sftp === null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!in_array(self::PROTOCOL, stream_get_wrappers(), true)) {
|
||||||
|
stream_wrapper_register(self::PROTOCOL, self::class);
|
||||||
|
}
|
||||||
|
$root = rtrim((string)$storage->getRoot(), '/');
|
||||||
|
$context = stream_context_create([self::PROTOCOL => [
|
||||||
|
'session' => $sftp,
|
||||||
|
'path' => $root . '/' . ltrim($path, '/'),
|
||||||
|
'storage' => $storage,
|
||||||
|
'file' => $path,
|
||||||
|
]]);
|
||||||
|
self::$openBroke = false;
|
||||||
|
$fh = @fopen(self::PROTOCOL . '://file', 'r', false, $context);
|
||||||
|
} catch (\Throwable) {
|
||||||
|
// nothing was sent yet
|
||||||
|
self::$failures++;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if ($fh === false && self::openBroke()) {
|
||||||
|
throw new BackendException('SFTP connection failed while opening ' . $path . ': ' . self::$openCause);
|
||||||
|
}
|
||||||
|
return $fh;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The storage's phpseclib connection, with its SFTP channel open, when it offers what these
|
||||||
|
* streams need; null otherwise.
|
||||||
|
*/
|
||||||
|
private static function connection(object $storage): ?object {
|
||||||
|
if (!is_callable([$storage, 'getConnection']) || !is_callable([$storage, 'getRoot'])) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
$sftp = $storage->getConnection();
|
||||||
|
// is_callable and get_object_vars from here only see what is public: phpseclib 3 has
|
||||||
|
// methods of these names neither public nor at all.
|
||||||
|
if (!is_object($sftp) || !is_callable([$sftp, '_send_sftp_packet']) || !is_callable([$sftp, '_get_sftp_packet'])
|
||||||
|
|| !is_callable([$sftp, '_realpath']) || !is_callable([$sftp, 'pwd'])
|
||||||
|
|| !array_key_exists('packet_type', get_object_vars($sftp))) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// phpseclib opens the SFTP channel on its first SFTP call, not at login: on a fresh
|
||||||
|
// connection a packet sent directly would go nowhere. pwd() opens it like any call does.
|
||||||
|
if ($sftp->pwd() === false) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return $sftp;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read many small files whole, BATCH at a time: the OPEN, READ and CLOSE requests of a batch
|
||||||
|
* all go out before their answers are read, so a batch costs a few round trips instead of
|
||||||
|
* four per file. Meant for files of up to a few hundred KiB (the repository's index files):
|
||||||
|
* every file is held in memory.
|
||||||
|
*
|
||||||
|
* @param list<string> $paths below the storage's root
|
||||||
|
* @return array<string, string|null>|null contents by path; null for a file this could not
|
||||||
|
* read (missing, or a server error), so the caller
|
||||||
|
* reads it the usual way; null for all when the
|
||||||
|
* connection does not offer what this needs
|
||||||
|
* @throws BackendException when the connection failed while requests were out: the storage's
|
||||||
|
* own stream must not be tried on it
|
||||||
|
*/
|
||||||
|
public static function readMany(object $storage, array $paths): ?array {
|
||||||
|
if (self::$failures >= self::MAX_FAILURES) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$sftp = self::connection($storage);
|
||||||
|
$root = $sftp === null ? '' : rtrim((string)$storage->getRoot(), '/');
|
||||||
|
} catch (\Throwable) {
|
||||||
|
self::$failures++;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if ($sftp === null || self::$noBatches?->offsetExists($sftp) === true) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
$result = array_fill_keys($paths, null);
|
||||||
|
foreach (array_chunk($paths, self::BATCH) as $batch) {
|
||||||
|
try {
|
||||||
|
[$read, $closed] = self::readBatch($sftp, $root, $batch);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
self::$failures++;
|
||||||
|
// Answers to requests already sent may still arrive, and files_external would take
|
||||||
|
// them for its own: end the connection rather than leave it out of step.
|
||||||
|
self::disconnect($sftp);
|
||||||
|
throw $e instanceof BackendException ? $e : new BackendException('SFTP connection failed while reading ' . count($batch) . ' files: ' . $e->getMessage(), 0, $e);
|
||||||
|
}
|
||||||
|
foreach ($read as $path => $contents) {
|
||||||
|
$result[$path] = $contents;
|
||||||
|
}
|
||||||
|
if (!$closed) {
|
||||||
|
// The server did not confirm closing these files: no more batches on this
|
||||||
|
// connection, so handles cannot pile up. The rest is read the usual way.
|
||||||
|
self::noMoreBatches($sftp);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self::$failures = 0;
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function noMoreBatches(object $sftp): void {
|
||||||
|
self::$noBatches ??= new \WeakMap();
|
||||||
|
self::$noBatches[$sftp] = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function disconnect(object $sftp): void {
|
||||||
|
try {
|
||||||
|
if (is_callable([$sftp, 'disconnect'])) {
|
||||||
|
$sftp->disconnect();
|
||||||
|
}
|
||||||
|
} catch (\Throwable) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Requests of one kind for a whole batch, then their answers in order. @return list<array{int, string}> */
|
||||||
|
private static function exchange(object $sftp, int $type, array $packets): array {
|
||||||
|
$ids = [];
|
||||||
|
foreach ($packets as $packet) {
|
||||||
|
$id = self::nextId();
|
||||||
|
if (!$sftp->_send_sftp_packet($type, $packet, $id)) {
|
||||||
|
throw new BackendException('SFTP connection failed while sending a request');
|
||||||
|
}
|
||||||
|
$ids[] = $id;
|
||||||
|
}
|
||||||
|
$answers = [];
|
||||||
|
foreach ($ids as $id) {
|
||||||
|
$response = $sftp->_get_sftp_packet($id);
|
||||||
|
if (!is_string($response)) {
|
||||||
|
throw new BackendException('SFTP connection failed while waiting for an answer');
|
||||||
|
}
|
||||||
|
$answers[] = [$sftp->packet_type, $response];
|
||||||
|
}
|
||||||
|
return $answers;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param list<string> $batch
|
||||||
|
* @return array{0: array<string, string>, 1: bool} the files read whole, and whether the
|
||||||
|
* server confirmed closing every handle
|
||||||
|
*/
|
||||||
|
private static function readBatch(object $sftp, string $root, array $batch): array {
|
||||||
|
$opens = [];
|
||||||
|
$openPaths = [];
|
||||||
|
foreach ($batch as $path) {
|
||||||
|
$remote = $sftp->_realpath($root . '/' . ltrim($path, '/'));
|
||||||
|
if (is_string($remote)) {
|
||||||
|
$opens[] = pack('Na*N2', strlen($remote), $remote, self::FXF_READ, 0);
|
||||||
|
$openPaths[] = $path;
|
||||||
|
}
|
||||||
|
// a path that cannot be resolved is left to the caller's usual read
|
||||||
|
}
|
||||||
|
$handles = []; // path => handle
|
||||||
|
foreach (self::exchange($sftp, self::FXP_OPEN, $opens) as $i => [$type, $response]) {
|
||||||
|
if ($type === self::FXP_HANDLE) {
|
||||||
|
$handles[$openPaths[$i]] = substr($response, 4);
|
||||||
|
}
|
||||||
|
// any other answer (missing, no permission): the caller reads it the usual way
|
||||||
|
}
|
||||||
|
$data = array_fill_keys(array_keys($handles), '');
|
||||||
|
$reading = $handles;
|
||||||
|
while ($reading !== []) {
|
||||||
|
$length = self::chunk($sftp);
|
||||||
|
$reads = [];
|
||||||
|
foreach ($reading as $path => $handle) {
|
||||||
|
$offset = strlen($data[$path]);
|
||||||
|
$reads[] = pack('Na*N3', strlen($handle), $handle, intdiv($offset, 4294967296), $offset % 4294967296, $length);
|
||||||
|
}
|
||||||
|
$paths = array_keys($reading);
|
||||||
|
foreach (self::exchange($sftp, self::FXP_READ, $reads) as $i => [$type, $response]) {
|
||||||
|
$path = $paths[$i];
|
||||||
|
$declared = strlen($response) >= 4 ? unpack('N', $response) : false;
|
||||||
|
if ($type === self::FXP_DATA && is_array($declared) && $declared[1] > 0 && $declared[1] === strlen($response) - 4
|
||||||
|
&& $declared[1] <= $length && strlen($data[$path]) + $declared[1] <= self::MANY_MAX_BYTES) {
|
||||||
|
$data[$path] .= substr($response, 4); // more may follow: read on from there
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
unset($reading[$path]);
|
||||||
|
$status = $type === self::FXP_STATUS && is_array($declared) ? $declared[1] : null;
|
||||||
|
if ($status !== self::FX_EOF) {
|
||||||
|
// a server error, a malformed answer or a file too large for this: the caller
|
||||||
|
// reads it the usual way
|
||||||
|
unset($data[$path]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$closes = [];
|
||||||
|
foreach ($handles as $handle) {
|
||||||
|
$closes[] = pack('Na*', strlen($handle), $handle);
|
||||||
|
}
|
||||||
|
$closed = true;
|
||||||
|
foreach (self::exchange($sftp, self::FXP_CLOSE, $closes) as [$type, $response]) {
|
||||||
|
$status = $type === self::FXP_STATUS && strlen($response) >= 4 ? unpack('N', $response) : false;
|
||||||
|
if (!is_array($status) || $status[1] !== self::FX_OK) {
|
||||||
|
$closed = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [$data, $closed];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test hook: forget the learned READ sizes and earlier failures. */
|
||||||
|
public static function reset(): void {
|
||||||
|
self::$chunks = null;
|
||||||
|
self::$noBatches = null;
|
||||||
|
self::$failures = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function stream_open(string $url, string $mode, int $options, ?string &$openedPath): bool {
|
||||||
|
$options = stream_context_get_options($this->context ?? stream_context_get_default())[self::PROTOCOL] ?? [];
|
||||||
|
if (!isset($options['session'], $options['path']) || !is_object($options['session'])) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
$this->sftp = $options['session'];
|
||||||
|
$storage = $options['storage'] ?? null;
|
||||||
|
$this->storage = is_object($storage) ? $storage : null;
|
||||||
|
$this->file = (string)($options['file'] ?? '');
|
||||||
|
try {
|
||||||
|
$remote = $this->sftp->_realpath((string)$options['path']);
|
||||||
|
if (!is_string($remote)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
$id = self::nextId();
|
||||||
|
if (!$this->sftp->_send_sftp_packet(self::FXP_OPEN, pack('Na*N2', strlen($remote), $remote, self::FXF_READ, 0), $id)) {
|
||||||
|
return $this->connectionFailed('the request could not be sent');
|
||||||
|
}
|
||||||
|
$response = $this->sftp->_get_sftp_packet($id);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return $this->connectionFailed(get_class($e) . ': ' . $e->getMessage());
|
||||||
|
}
|
||||||
|
if (!is_string($response)) {
|
||||||
|
return $this->connectionFailed('no answer');
|
||||||
|
}
|
||||||
|
if ($this->sftp->packet_type !== self::FXP_HANDLE) {
|
||||||
|
return false; // an answer, but no handle: the file cannot be opened
|
||||||
|
}
|
||||||
|
$this->handle = substr($response, 4);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Not a missing file but a connection that does not behave as expected. */
|
||||||
|
private function connectionFailed(string $cause): bool {
|
||||||
|
self::$failures++;
|
||||||
|
self::$openBroke = true;
|
||||||
|
self::$openCause = $cause . self::connectionState($this->sftp);
|
||||||
|
// unknown state: closed, so that nothing else uses it (and the backend can build a new one)
|
||||||
|
self::disconnect($this->sftp);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What phpseclib knows about the connection, for an error message. */
|
||||||
|
private static function connectionState(object $sftp): string {
|
||||||
|
$state = [];
|
||||||
|
try {
|
||||||
|
if (is_callable([$sftp, 'isConnected'])) {
|
||||||
|
$state[] = 'connected: ' . ($sftp->isConnected() ? 'yes' : 'no');
|
||||||
|
}
|
||||||
|
if (is_callable([$sftp, 'getLastError'])) {
|
||||||
|
$last = (string)$sftp->getLastError();
|
||||||
|
if ($last !== '') {
|
||||||
|
$state[] = 'ssh: ' . $last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (is_callable([$sftp, 'getLastSFTPError'])) {
|
||||||
|
$last = (string)$sftp->getLastSFTPError();
|
||||||
|
if ($last !== '') {
|
||||||
|
$state[] = 'sftp: ' . $last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (\Throwable) {
|
||||||
|
}
|
||||||
|
return $state === [] ? '' : ' (' . implode('; ', $state) . ')';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether the last stream_open() failed on the connection; a call, as fopen() sets it. */
|
||||||
|
private static function openBroke(): bool {
|
||||||
|
return self::$openBroke;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function stream_read(int $count): string|false {
|
||||||
|
if ($this->fallback !== null) {
|
||||||
|
return fread($this->fallback, $count);
|
||||||
|
}
|
||||||
|
while (strlen($this->buffer) < $count && !$this->eof) {
|
||||||
|
if (!$this->receive()) {
|
||||||
|
return $this->useFallback() ? $this->stream_read($count) : false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$data = substr($this->buffer, 0, $count);
|
||||||
|
$this->buffer = substr($this->buffer, strlen($data));
|
||||||
|
$this->pos += strlen($data);
|
||||||
|
return $data;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function stream_eof(): bool {
|
||||||
|
if ($this->fallback !== null) {
|
||||||
|
return feof($this->fallback);
|
||||||
|
}
|
||||||
|
return $this->eof && $this->buffer === '';
|
||||||
|
}
|
||||||
|
|
||||||
|
public function stream_tell(): int {
|
||||||
|
if ($this->fallback !== null) {
|
||||||
|
$pos = ftell($this->fallback);
|
||||||
|
return $pos === false ? $this->pos : $pos;
|
||||||
|
}
|
||||||
|
return $this->pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function stream_seek(int $offset, int $whence = SEEK_SET): bool {
|
||||||
|
if ($this->fallback !== null) {
|
||||||
|
return fseek($this->fallback, $offset, $whence) === 0;
|
||||||
|
}
|
||||||
|
if ($whence === SEEK_CUR) {
|
||||||
|
$offset += $this->pos;
|
||||||
|
} elseif ($whence !== SEEK_SET) {
|
||||||
|
return false; // the size is never asked for, to save a round trip
|
||||||
|
}
|
||||||
|
if ($offset < 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if ($this->failed) {
|
||||||
|
$this->pos = $offset;
|
||||||
|
return $this->useFallback();
|
||||||
|
}
|
||||||
|
if ($offset >= $this->pos && $offset - $this->pos <= strlen($this->buffer)) {
|
||||||
|
$this->buffer = substr($this->buffer, $offset - $this->pos);
|
||||||
|
} else {
|
||||||
|
$this->drain();
|
||||||
|
$this->buffer = '';
|
||||||
|
$this->next = $offset;
|
||||||
|
$this->eof = false;
|
||||||
|
$this->depth = 1;
|
||||||
|
$this->shortAnswer = null;
|
||||||
|
}
|
||||||
|
$this->pos = $offset;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return array<int|string, int> */
|
||||||
|
public function stream_stat(): array {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
public function stream_close(): void {
|
||||||
|
if (!$this->failed) {
|
||||||
|
self::$failures = 0;
|
||||||
|
}
|
||||||
|
$fallback = $this->fallback;
|
||||||
|
$this->fallback = null;
|
||||||
|
if ($fallback !== null) {
|
||||||
|
fclose($fallback);
|
||||||
|
}
|
||||||
|
$this->closeHandle();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Go on with the storage's own stream from the current position, after a server error on this
|
||||||
|
* one. The data still buffered is dropped and read again. False after a connection error, or
|
||||||
|
* when that stream cannot be opened or positioned: then the read fails as it would have
|
||||||
|
* without this class. Reopening by path is safe for a repository: packs and index files are
|
||||||
|
* never rewritten, and every object is authenticated when it is decrypted.
|
||||||
|
*/
|
||||||
|
private function useFallback(): bool {
|
||||||
|
$this->closeHandle();
|
||||||
|
$this->buffer = '';
|
||||||
|
// files_external's stream takes whatever answer comes next on the connection: after a
|
||||||
|
// connection error, answers to this stream's requests may still be on their way.
|
||||||
|
if ($this->broken || $this->storage === null || !is_callable([$this->storage, 'fopen'])) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$fh = $this->storage->fopen($this->file, 'r');
|
||||||
|
if (!is_resource($fh)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if ($this->pos > 0 && fseek($fh, $this->pos) !== 0) {
|
||||||
|
fclose($fh);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} catch (\Throwable) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
$this->fallback = $fh;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function closeHandle(): void {
|
||||||
|
$this->drain();
|
||||||
|
if ($this->handle !== '' && !$this->broken) {
|
||||||
|
try {
|
||||||
|
$id = self::nextId();
|
||||||
|
if (!$this->sftp->_send_sftp_packet(self::FXP_CLOSE, pack('Na*', strlen($this->handle), $this->handle), $id)
|
||||||
|
|| !is_string($this->sftp->_get_sftp_packet($id))) {
|
||||||
|
$this->broken = true;
|
||||||
|
}
|
||||||
|
} catch (\Throwable) {
|
||||||
|
$this->broken = true;
|
||||||
|
}
|
||||||
|
if ($this->broken) {
|
||||||
|
self::disconnect($this->sftp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$this->handle = '';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Keep the pipeline full, then take the oldest answer. False on a connection or server error. */
|
||||||
|
private function receive(): bool {
|
||||||
|
if ($this->failed) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return $this->receiveOne();
|
||||||
|
} catch (\Throwable) {
|
||||||
|
$this->broken = true;
|
||||||
|
return $this->fail();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function receiveOne(): bool {
|
||||||
|
while (count($this->inFlight) < $this->depth) {
|
||||||
|
$id = self::nextId();
|
||||||
|
$length = self::chunk($this->sftp);
|
||||||
|
$packet = pack('Na*N3', strlen($this->handle), $this->handle, intdiv($this->next, 4294967296), $this->next % 4294967296, $length);
|
||||||
|
if (!$this->sftp->_send_sftp_packet(self::FXP_READ, $packet, $id)) {
|
||||||
|
$this->broken = true;
|
||||||
|
return $this->fail();
|
||||||
|
}
|
||||||
|
$this->inFlight[] = [$id, $this->next, $length];
|
||||||
|
$this->next += $length;
|
||||||
|
}
|
||||||
|
[$id, $offset, $length] = array_shift($this->inFlight);
|
||||||
|
$response = $this->sftp->_get_sftp_packet($id);
|
||||||
|
if (!is_string($response)) {
|
||||||
|
$this->broken = true;
|
||||||
|
return $this->fail();
|
||||||
|
}
|
||||||
|
if ($this->sftp->packet_type === self::FXP_DATA) {
|
||||||
|
$data = substr($response, 4);
|
||||||
|
if ($data === '') {
|
||||||
|
return $this->fail();
|
||||||
|
}
|
||||||
|
$this->buffer .= $data;
|
||||||
|
$this->depth = min(self::DEPTH, $this->depth * 2);
|
||||||
|
if ($this->shortAnswer !== null) {
|
||||||
|
// More data after a short answer: that one was the server's cap, not the end.
|
||||||
|
// Never below what every server must support: a one-off small answer is no cap.
|
||||||
|
self::$chunks ??= new \WeakMap();
|
||||||
|
self::$chunks[$this->sftp] = max(self::MIN_CHUNK, min(self::chunk($this->sftp), $this->shortAnswer));
|
||||||
|
$this->shortAnswer = null;
|
||||||
|
}
|
||||||
|
if (strlen($data) < $length) {
|
||||||
|
// Fewer bytes than asked: the end of the file, or a server that caps a READ. The
|
||||||
|
// requests behind this one assumed a full answer, so drop them and ask again from here.
|
||||||
|
$this->drain();
|
||||||
|
$this->next = $offset + strlen($data);
|
||||||
|
$this->shortAnswer = strlen($data);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
$status = $this->sftp->packet_type === self::FXP_STATUS ? unpack('N', $response) : false;
|
||||||
|
if (is_array($status) && ($status[1] ?? null) === self::FX_EOF) {
|
||||||
|
$this->eof = true;
|
||||||
|
$this->shortAnswer = null;
|
||||||
|
$this->drain();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return $this->fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
private function fail(): bool {
|
||||||
|
$this->drain();
|
||||||
|
if (!$this->failed) {
|
||||||
|
self::$failures++;
|
||||||
|
}
|
||||||
|
$this->failed = true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read and drop the answers still on their way, so the connection is free again. */
|
||||||
|
private function drain(): void {
|
||||||
|
$inFlight = $this->inFlight;
|
||||||
|
$this->inFlight = [];
|
||||||
|
try {
|
||||||
|
foreach ($this->broken ? [] : $inFlight as [$id]) {
|
||||||
|
if (!is_string($this->sftp->_get_sftp_packet($id))) {
|
||||||
|
$this->broken = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (\Throwable) {
|
||||||
|
$this->broken = true;
|
||||||
|
}
|
||||||
|
if ($this->broken) {
|
||||||
|
// unknown state: closed, so that nothing else uses it (and the backend can build a new one)
|
||||||
|
self::disconnect($this->sftp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function chunk(object $sftp): int {
|
||||||
|
$learned = self::$chunks?->offsetExists($sftp) === true ? self::$chunks->offsetGet($sftp) : null;
|
||||||
|
return is_int($learned) ? $learned : self::CHUNK;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function nextId(): int {
|
||||||
|
self::$requestId = self::$requestId >= 0x7fffffff ? 0x4e470000 : self::$requestId + 1;
|
||||||
|
return self::$requestId;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,20 +17,43 @@ use OCP\Files\Storage\IWriteStreamStorage;
|
|||||||
final class StorageBackend implements IBackend {
|
final class StorageBackend implements IBackend {
|
||||||
/** Object stores write whole objects atomically; other storages get a temp name + rename. */
|
/** Object stores write whole objects atomically; other storages get a temp name + rename. */
|
||||||
private bool $atomicPut;
|
private bool $atomicPut;
|
||||||
|
/** SFTP: read through SftpReadStream, which keeps several requests in flight. */
|
||||||
|
private bool $sftpReads;
|
||||||
|
/** Seconds without a request after which an SFTP connection is checked before it is used. */
|
||||||
|
public const IDLE_CHECK = 60;
|
||||||
|
private float $lastUsed;
|
||||||
|
/** Times the storage was built anew after its connection died (for tests and diagnostics). */
|
||||||
|
public int $reconnects = 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param (\Closure():IStorage)|null $reconnect builds the same storage anew, with a new
|
||||||
|
* connection
|
||||||
|
*/
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private IStorage $storage,
|
private IStorage $storage,
|
||||||
private string $base = '',
|
private string $base = '',
|
||||||
|
bool $createBase = true,
|
||||||
|
private ?\Closure $reconnect = null,
|
||||||
) {
|
) {
|
||||||
|
$this->lastUsed = microtime(true);
|
||||||
$this->base = trim($base, '/');
|
$this->base = trim($base, '/');
|
||||||
self::validate($this->base, true);
|
self::validate($this->base, true);
|
||||||
$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
|
$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
|
||||||
if ($this->base !== '') {
|
$this->sftpReads = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\SFTP::class);
|
||||||
|
if ($createBase && $this->base !== '') {
|
||||||
$this->mkdirs($this->base);
|
$this->mkdirs($this->base);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A temporary upload of put(): "<name>.part-<8 hex>". */
|
||||||
|
public static function isTemporary(string $name): bool {
|
||||||
|
return preg_match('/\.part-[0-9a-f]{8}$/', $name) === 1;
|
||||||
|
}
|
||||||
|
|
||||||
public function put(string $path, $stream): void {
|
public function put(string $path, $stream): void {
|
||||||
|
// not repeated on a new connection: the stream is (partly) read
|
||||||
|
$this->idleCheck();
|
||||||
|
$expected = self::remaining($stream);
|
||||||
$target = $this->abs($path);
|
$target = $this->abs($path);
|
||||||
$this->mkdirs(dirname($target));
|
$this->mkdirs(dirname($target));
|
||||||
$writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4));
|
$writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4));
|
||||||
@@ -54,6 +77,10 @@ final class StorageBackend implements IBackend {
|
|||||||
if (is_resource($stream)) {
|
if (is_resource($stream)) {
|
||||||
fclose($stream);
|
fclose($stream);
|
||||||
}
|
}
|
||||||
|
// Some adapters report success after a short write or a failed flush: check the size.
|
||||||
|
if ($expected !== null && $this->storage->filesize($writeTo) !== $expected) {
|
||||||
|
throw new BackendException('Incomplete write for ' . $path);
|
||||||
|
}
|
||||||
if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) {
|
if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) {
|
||||||
throw new BackendException('Cannot finalise ' . $path);
|
throw new BackendException('Cannot finalise ' . $path);
|
||||||
}
|
}
|
||||||
@@ -69,6 +96,19 @@ final class StorageBackend implements IBackend {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function get(string $path) {
|
public function get(string $path) {
|
||||||
|
return $this->call(fn () => $this->open($path));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return resource */
|
||||||
|
private function open(string $path) {
|
||||||
|
if ($this->sftpReads) {
|
||||||
|
$fh = SftpReadStream::open($this->storage, $this->abs($path));
|
||||||
|
if (is_resource($fh)) {
|
||||||
|
return $fh;
|
||||||
|
}
|
||||||
|
// null: not supported or failing for now; false: not opened, which need not mean
|
||||||
|
// missing. Either way the storage's own stream decides.
|
||||||
|
}
|
||||||
$fh = $this->storage->fopen($this->abs($path), 'r');
|
$fh = $this->storage->fopen($this->abs($path), 'r');
|
||||||
if ($fh === false) {
|
if ($fh === false) {
|
||||||
throw new BackendException('Not found: ' . $path);
|
throw new BackendException('Not found: ' . $path);
|
||||||
@@ -76,7 +116,40 @@ final class StorageBackend implements IBackend {
|
|||||||
return $fh;
|
return $fh;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Over SFTP several at a time (SftpReadStream::readMany()); elsewhere one after the other. */
|
||||||
|
public function getMany(array $paths): \Generator {
|
||||||
|
foreach (array_chunk($paths, 8 * SftpReadStream::BATCH) as $chunk) {
|
||||||
|
$fast = null;
|
||||||
|
if ($this->sftpReads) {
|
||||||
|
$abs = array_map(fn (string $p): string => $this->abs($p), $chunk);
|
||||||
|
$fast = $this->call(fn () => SftpReadStream::readMany($this->storage, $abs));
|
||||||
|
$fast = $fast === null ? null : array_combine($chunk, array_map(fn (string $a): ?string => $fast[$a] ?? null, $abs));
|
||||||
|
}
|
||||||
|
foreach ($chunk as $path) {
|
||||||
|
$data = $fast[$path] ?? null;
|
||||||
|
if ($data === null) {
|
||||||
|
// not read in the batch (or no batches here): the usual way, which also
|
||||||
|
// reports a missing file as such
|
||||||
|
$fh = $this->get($path);
|
||||||
|
try {
|
||||||
|
$data = stream_get_contents($fh);
|
||||||
|
} finally {
|
||||||
|
fclose($fh);
|
||||||
|
}
|
||||||
|
if ($data === false) {
|
||||||
|
throw new BackendException('Read failed for ' . $path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
yield $path => $data;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public function getRange(string $path, int $offset, int $length): string {
|
public function getRange(string $path, int $offset, int $length): string {
|
||||||
|
return $this->call(fn () => $this->readRange($path, $offset, $length));
|
||||||
|
}
|
||||||
|
|
||||||
|
private function readRange(string $path, int $offset, int $length): string {
|
||||||
$fh = $this->get($path);
|
$fh = $this->get($path);
|
||||||
try {
|
try {
|
||||||
if (fseek($fh, $offset) !== 0) {
|
if (fseek($fh, $offset) !== 0) {
|
||||||
@@ -99,22 +172,162 @@ final class StorageBackend implements IBackend {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Whether the storage accepts the connection settings (files_external's own check). */
|
||||||
|
public function canConnect(): bool {
|
||||||
|
try {
|
||||||
|
return !method_exists($this->storage, 'test') || $this->storage->test();
|
||||||
|
} catch (\Throwable) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public function exists(string $path): bool {
|
public function exists(string $path): bool {
|
||||||
return $this->storage->file_exists($this->abs($path));
|
$abs = $this->abs($path);
|
||||||
|
return $this->call(fn () => $this->storage->file_exists($abs), fn (bool $found) => !$found);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function list(string $prefix): array {
|
public function list(string $prefix): array {
|
||||||
|
$abs = $this->abs($prefix);
|
||||||
|
$result = $this->call(function () use ($abs): array {
|
||||||
$result = [];
|
$result = [];
|
||||||
$this->listInto($this->abs($prefix), $result);
|
$this->listInto($abs, $result);
|
||||||
|
return $result;
|
||||||
|
}, fn (array $found) => $found === []);
|
||||||
sort($result);
|
sort($result);
|
||||||
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
||||||
return array_map(fn ($p) => substr($p, $cut), $result);
|
return array_map(fn ($p) => substr($p, $cut), $result);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function delete(string $path): void {
|
public function delete(string $path): void {
|
||||||
|
$this->idleCheck();
|
||||||
$this->storage->unlink($this->abs($path));
|
$this->storage->unlink($this->abs($path));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function move(string $from, string $to): void {
|
||||||
|
// not repeated on a new connection: the server may have done it before the connection died
|
||||||
|
$this->idleCheck();
|
||||||
|
$target = $this->abs($to);
|
||||||
|
$this->mkdirs(dirname($target));
|
||||||
|
if (!$this->storage->rename($this->abs($from), $target)) {
|
||||||
|
throw new BackendException("Cannot move $from to $to");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function listFiles(string $dir): array {
|
||||||
|
$abs = $this->abs($dir);
|
||||||
|
return $this->call(fn () => $this->filesIn($dir, $abs), fn (array $found) => $found === []);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return list<string> */
|
||||||
|
private function filesIn(string $dir, string $abs): array {
|
||||||
|
if (!$this->storage->is_dir($abs)) {
|
||||||
|
return []; // not created yet
|
||||||
|
}
|
||||||
|
$dh = $this->storage->opendir($abs);
|
||||||
|
if ($dh === false) {
|
||||||
|
// An existing folder read as empty would look like a new repository.
|
||||||
|
throw new BackendException('Cannot list ' . $dir);
|
||||||
|
}
|
||||||
|
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
||||||
|
$result = [];
|
||||||
|
while (($name = readdir($dh)) !== false) {
|
||||||
|
if ($name !== '.' && $name !== '..' && !self::isTemporary($name)) {
|
||||||
|
$result[] = substr($abs . '/' . $name, $cut);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
closedir($dh);
|
||||||
|
sort($result);
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run $op; when it throws, or $failed says its result may come from a dead connection, and
|
||||||
|
* the SFTP connection turns out to be dead, run it once more on a new connection.
|
||||||
|
*
|
||||||
|
* @template T
|
||||||
|
* @param \Closure():T $op
|
||||||
|
* @param (\Closure(T):bool)|null $failed
|
||||||
|
* @return T
|
||||||
|
*/
|
||||||
|
private function call(\Closure $op, ?\Closure $failed = null): mixed {
|
||||||
|
$this->idleCheck();
|
||||||
|
try {
|
||||||
|
$result = $op();
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
if (!$this->reopenIfDead()) {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
return $op();
|
||||||
|
}
|
||||||
|
if ($failed !== null && $failed($result) && $this->reopenIfDead()) {
|
||||||
|
return $op();
|
||||||
|
}
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A Hetzner Storage Box drops an SFTP connection that carried a lot of data and then stayed
|
||||||
|
* quiet for a few minutes, while phpseclib still takes it for open; files_external never
|
||||||
|
* builds a new one, and every request on it fails as if the file were missing. So after
|
||||||
|
* such a pause the connection is checked (one round trip) before it is used again.
|
||||||
|
*/
|
||||||
|
private function idleCheck(): void {
|
||||||
|
$now = microtime(true);
|
||||||
|
if ($this->reconnect !== null && $this->sftpReads && $now - $this->lastUsed >= self::IDLE_CHECK
|
||||||
|
&& $this->alive(true) === false) {
|
||||||
|
$this->reopen();
|
||||||
|
}
|
||||||
|
$this->lastUsed = microtime(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function reopenIfDead(): bool {
|
||||||
|
if ($this->reconnect === null || !$this->sftpReads || $this->alive(false) !== false) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
$this->reopen();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether the storage's SFTP connection is alive; null when it cannot tell.
|
||||||
|
* $ask: send a request instead of trusting what phpseclib saw last.
|
||||||
|
*/
|
||||||
|
private function alive(bool $ask): ?bool {
|
||||||
|
try {
|
||||||
|
if (!is_callable([$this->storage, 'getConnection'])) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
$sftp = $this->storage->getConnection();
|
||||||
|
if (!is_object($sftp) || !is_callable([$sftp, 'isConnected'])) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!$sftp->isConnected()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!$ask || !is_callable([$sftp, 'stat'])) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (is_callable([$sftp, 'clearStatCache'])) {
|
||||||
|
$sftp->clearStatCache(); // or stat() answers from memory
|
||||||
|
}
|
||||||
|
$sftp->stat('.');
|
||||||
|
return (bool)$sftp->isConnected();
|
||||||
|
} catch (\Throwable) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function reopen(): void {
|
||||||
|
/** @var \Closure():IStorage $reconnect */
|
||||||
|
$reconnect = $this->reconnect;
|
||||||
|
$this->storage = $reconnect();
|
||||||
|
$this->reconnects++;
|
||||||
|
$this->lastUsed = microtime(true);
|
||||||
|
if (function_exists('OCP\Log\logger')) {
|
||||||
|
\OCP\Log\logger('ng_backup')->info('The SFTP connection to the backup location was lost; reconnected');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private function listInto(string $dir, array &$result): void {
|
private function listInto(string $dir, array &$result): void {
|
||||||
if (!$this->storage->is_dir($dir)) {
|
if (!$this->storage->is_dir($dir)) {
|
||||||
return;
|
return;
|
||||||
@@ -124,19 +337,39 @@ final class StorageBackend implements IBackend {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
while (($name = readdir($dh)) !== false) {
|
while (($name = readdir($dh)) !== false) {
|
||||||
if ($name === '.' || $name === '..' || str_contains($name, '.part-')) {
|
if ($name === '.' || $name === '..') {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
$child = $dir . '/' . $name;
|
$child = $dir . '/' . $name;
|
||||||
if ($this->storage->is_dir($child)) {
|
if ($this->storage->is_dir($child)) {
|
||||||
$this->listInto($child, $result);
|
$this->listInto($child, $result);
|
||||||
} else {
|
} elseif (!self::isTemporary($name)) {
|
||||||
$result[] = $child;
|
$result[] = $child;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
closedir($dh);
|
closedir($dh);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bytes left to read in a local $stream (php://temp, php://memory or a plain file), or null:
|
||||||
|
* stream wrappers of remote storages do not report a reliable size.
|
||||||
|
*/
|
||||||
|
public static function remaining(mixed $stream): ?int {
|
||||||
|
if (!is_resource($stream)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
$meta = stream_get_meta_data($stream);
|
||||||
|
if (!($meta['seekable'] ?? false) || !in_array($meta['wrapper_type'] ?? '', ['PHP', 'plainfile'], true)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
$stat = fstat($stream);
|
||||||
|
$pos = ftell($stream);
|
||||||
|
if ($stat === false || $pos === false) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return max(0, $stat['size'] - $pos);
|
||||||
|
}
|
||||||
|
|
||||||
private function mkdirs(string $dir): void {
|
private function mkdirs(string $dir): void {
|
||||||
if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) {
|
if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -22,10 +22,12 @@ use Psr\Log\LoggerInterface;
|
|||||||
* With system cron a step may take minutes; with webcron/AJAX cron it is kept short so it fits
|
* With system cron a step may take minutes; with webcron/AJAX cron it is kept short so it fits
|
||||||
* in a normal web request (hosted Nextcloud).
|
* in a normal web request (hosted Nextcloud).
|
||||||
*/
|
*/
|
||||||
class BackupJob extends TimedJob {
|
final class BackupJob extends TimedJob {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
ITimeFactory $time,
|
ITimeFactory $time,
|
||||||
private BackupService $backups,
|
private BackupService $backups,
|
||||||
|
private \OCA\NgBackup\Service\PruneService $prune,
|
||||||
|
private \OCA\NgBackup\Service\AlertService $alerts,
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RunMapper $runs,
|
private RunMapper $runs,
|
||||||
private IAppConfig $appConfig,
|
private IAppConfig $appConfig,
|
||||||
@@ -36,15 +38,63 @@ class BackupJob extends TimedJob {
|
|||||||
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
|
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private const MAX_DAILY_ATTEMPTS = 3;
|
||||||
|
|
||||||
protected function run($argument): void {
|
protected function run($argument): void {
|
||||||
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
|
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
|
||||||
$deadline = microtime(true) + $budget;
|
$deadline = microtime(true) + $budget;
|
||||||
$this->startScheduled();
|
$this->startScheduled();
|
||||||
foreach ($this->runs->findRunning() as $run) {
|
foreach ($this->runs->findRunning(null, BackupService::KIND_FULL) as $run) {
|
||||||
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
|
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
|
||||||
$run = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
$next = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
||||||
|
if ($next === $run) {
|
||||||
|
break; // step() hands back the same object when another process holds the run's lease
|
||||||
|
}
|
||||||
|
$run = $next;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
$this->failStaleRestores();
|
||||||
|
$this->weeklyPrune();
|
||||||
|
foreach ($this->targets->list() as $t) {
|
||||||
|
$this->alerts->checkStale($t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A restore run that has not moved for 6 hours is considered dead (it would block cleanup forever). */
|
||||||
|
private function failStaleRestores(): void {
|
||||||
|
foreach ($this->runs->findRunning(null, 'restore') as $run) {
|
||||||
|
if (time() - $run->getUpdatedAt() > 6 * 3600) {
|
||||||
|
$run->setStatus(\OCA\NgBackup\Db\Run::FAILED);
|
||||||
|
$run->setError('No progress for 6 hours; marked as failed');
|
||||||
|
$run->setFinishedAt(time());
|
||||||
|
$this->runs->update($run);
|
||||||
|
$this->alerts->runFailed($run, '#' . $run->getTargetId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Once a week, apply the retention policy to non-append-only locations: from Sunday on, at the
|
||||||
|
* first moment no backup is running, so a long Sunday backup only delays it instead of skipping
|
||||||
|
* the week.
|
||||||
|
*/
|
||||||
|
private function weeklyPrune(): void {
|
||||||
|
$lastWeek = $this->appConfig->getValueString(Application::APP_ID, 'last_prune_week', '');
|
||||||
|
$due = date('N') === '7' ? date('o-W') : date('o-W', (int)strtotime('last sunday'));
|
||||||
|
if (strcmp($lastWeek, $due) >= 0 || $this->runs->findRunning(null, BackupService::KIND_FULL) !== []) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
foreach ($this->targets->list() as $t) {
|
||||||
|
if ($t->getAppendOnly()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$this->prune->apply($t, false, true);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$this->logger->warning('NG Backup: cleanup of ' . $t->getName() . ' failed: ' . $e->getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', $due);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
|
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
|
||||||
@@ -53,13 +103,19 @@ class BackupJob extends TimedJob {
|
|||||||
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
|
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
$todayAt = strtotime(date('Y-m-d') . ' ' . $at);
|
$todayAt = (int)strtotime(date('Y-m-d') . ' ' . $at);
|
||||||
if (time() < $todayAt) {
|
if (time() < $todayAt) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
foreach ($this->targets->list() as $t) {
|
foreach ($this->targets->list() as $t) {
|
||||||
$last = $this->runs->lastFinished($t->getId(), BackupService::KIND_FULL);
|
$last = $this->runs->lastFinished($t->getId(), BackupService::KIND_FULL);
|
||||||
if ($this->runs->findRunning($t->getId()) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
|
if ($this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// After a failure: retry at most MAX_DAILY_ATTEMPTS times a day, an hour apart, instead
|
||||||
|
// of starting (and notifying about) a new full run every five minutes all day.
|
||||||
|
$today = $this->runs->startedSince($t->getId(), BackupService::KIND_FULL, $todayAt);
|
||||||
|
if (count($today) >= self::MAX_DAILY_ATTEMPTS || ($today !== [] && time() - (int)$today[0]->getFinishedAt() < 3600)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\BackgroundJob;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Db\Run;
|
||||||
|
use OCA\NgBackup\Db\RunMapper;
|
||||||
|
use OCA\NgBackup\Service\RestoreService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCP\AppFramework\Utility\ITimeFactory;
|
||||||
|
use OCP\BackgroundJob\QueuedJob;
|
||||||
|
use Psr\Log\LoggerInterface;
|
||||||
|
|
||||||
|
/** Runs a restore requested from the web interface, outside the web request. */
|
||||||
|
final class RestoreJob extends QueuedJob {
|
||||||
|
public function __construct(
|
||||||
|
ITimeFactory $time,
|
||||||
|
private RunMapper $runs,
|
||||||
|
private TargetService $targets,
|
||||||
|
private RestoreService $restore,
|
||||||
|
private LoggerInterface $logger,
|
||||||
|
private \OCA\NgBackup\Service\AlertService $alerts,
|
||||||
|
) {
|
||||||
|
parent::__construct($time);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function run($argument): void {
|
||||||
|
$runId = (int)($argument['run'] ?? 0);
|
||||||
|
try {
|
||||||
|
$run = $this->runs->find($runId);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$this->logger->error('NG Backup restore job: run ' . $runId . ' not found', ['exception' => $e]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ($run->getStatus() !== Run::RUNNING) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$req = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
$run->setPhase('restoring');
|
||||||
|
$run->setUpdatedAt(time());
|
||||||
|
$run = $this->runs->update($run);
|
||||||
|
@set_time_limit(0);
|
||||||
|
$lastBeat = time();
|
||||||
|
$result = $this->restore->restoreUserFiles($this->targets->get((string)$run->getTargetId()), $req['snapshot'], $req['path'], $req['mode'],
|
||||||
|
function (int $done, int $total) use (&$run, &$lastBeat): void {
|
||||||
|
// Heartbeat, so a long restore is not mistaken for a dead one.
|
||||||
|
if (time() - $lastBeat >= 60) {
|
||||||
|
$lastBeat = time();
|
||||||
|
$run->setPhase("restoring $done/$total");
|
||||||
|
$run->setUpdatedAt(time());
|
||||||
|
$run = $this->runs->update($run);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
$run->setStatus(Run::DONE);
|
||||||
|
$run->setPhase('done');
|
||||||
|
$run->setStats(json_encode($result, JSON_THROW_ON_ERROR));
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$this->logger->error('NG Backup restore ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
||||||
|
$run->setStatus(Run::FAILED);
|
||||||
|
$run->setError($e->getMessage());
|
||||||
|
try {
|
||||||
|
$this->alerts->runFailed($run, '#' . $run->getTargetId());
|
||||||
|
} catch (\Throwable) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$run->setFinishedAt(time());
|
||||||
|
$run->setUpdatedAt(time());
|
||||||
|
$this->runs->update($run);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCA\NgBackup\Service\UserBackupService;
|
||||||
|
use OCA\NgBackup\UserMigration\UserMigrationInstaller;
|
||||||
|
use OCP\IUserManager;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class BackupUser extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private UserBackupService $userBackup,
|
||||||
|
private IUserManager $userManager,
|
||||||
|
private UserMigrationInstaller $installer,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:user:backup')
|
||||||
|
->setDescription('Back up one user via user_migration (account, settings, files and everything else a migrator covers)')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addArgument('user', InputArgument::REQUIRED, 'User id')
|
||||||
|
->addOption('install-user-migration', null, InputOption::VALUE_NONE, 'Install and enable user_migration if it is missing (non-interactive consent)')
|
||||||
|
->setHelp(<<<'HELP'
|
||||||
|
Streams a user_migration export straight into the encrypted repository: no zip, no copy in the
|
||||||
|
user's own storage. Shares data with the regular backup; this needs the user_migration app
|
||||||
|
enabled. The manifest path this prints is what occ backup:user:restore expects.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
occ backup:user:backup offsite alice
|
||||||
|
HELP);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$uid = (string)$input->getArgument('user');
|
||||||
|
$user = $this->userManager->get($uid);
|
||||||
|
if ($user === null) {
|
||||||
|
$output->writeln("<error>No such user: $uid</error>");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$target = $this->targets->get($input->getArgument('target'));
|
||||||
|
$this->installer->ensureEnabled($input, $output);
|
||||||
|
$manifest = $this->userBackup->backupUser($target, $user);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$output->writeln("Backed up <info>$uid</info> to $manifest.");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputArgument;
|
|||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class Browse extends Command {
|
final class Browse extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RestoreService $restore,
|
private RestoreService $restore,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use Symfony\Component\Console\Input\InputOption;
|
|||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
use Symfony\Component\Console\Question\Question;
|
use Symfony\Component\Console\Question\Question;
|
||||||
|
|
||||||
class KeyConfirm extends Command {
|
final class KeyConfirm extends Command {
|
||||||
public const STATEMENT = 'I have downloaded the recovery kit and stored it safely, outside this server. '
|
public const STATEMENT = 'I have downloaded the recovery kit and stored it safely, outside this server. '
|
||||||
. 'I understand that keeping these keys safe is my own responsibility and that backups cannot be restored without them.';
|
. 'I understand that keeping these keys safe is my own responsibility and that backups cannot be restored without them.';
|
||||||
public const CONFIRMATION = 'Yes, I confirm';
|
public const CONFIRMATION = 'Yes, I confirm';
|
||||||
@@ -28,7 +28,9 @@ class KeyConfirm extends Command {
|
|||||||
protected function configure(): void {
|
protected function configure(): void {
|
||||||
$this->setName('backup:key:confirm')
|
$this->setName('backup:key:confirm')
|
||||||
->setDescription('Confirm that the recovery kit is stored safely (required before the first backup)')
|
->setDescription('Confirm that the recovery kit is stored safely (required before the first backup)')
|
||||||
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Admin user id recorded with the confirmation', 'occ');
|
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Admin user id recorded with the confirmation', 'occ')
|
||||||
|
->addOption('code', null, InputOption::VALUE_REQUIRED, 'Confirmation code from the kit file (for scripts; asked otherwise)')
|
||||||
|
->addOption('confirm', null, InputOption::VALUE_REQUIRED, 'The confirmation phrase "' . self::CONFIRMATION . '" (for scripts; asked otherwise)');
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
@@ -40,7 +42,12 @@ class KeyConfirm extends Command {
|
|||||||
$output->writeln('');
|
$output->writeln('');
|
||||||
$output->writeln(wordwrap(self::STATEMENT, 78));
|
$output->writeln(wordwrap(self::STATEMENT, 78));
|
||||||
$output->writeln('');
|
$output->writeln('');
|
||||||
$answer = (string)$this->getHelper('question')->ask($input, $output, new Question('Type "' . self::CONFIRMATION . '" to proceed: '));
|
$code = $input->getOption('code') ?? (string)$this->getHelper('question')->ask($input, $output, new Question('Confirmation code from the recovery kit file ("confirmation_code"): '));
|
||||||
|
if (!$this->keys->checkKitCode($code)) {
|
||||||
|
$output->writeln('<error>That code does not match the current recovery kit (download it with occ backup:key:kit first).</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$answer = $input->getOption('confirm') ?? (string)$this->getHelper('question')->ask($input, $output, new Question('Type "' . self::CONFIRMATION . '" to proceed: '));
|
||||||
if (trim($answer) !== self::CONFIRMATION) {
|
if (trim($answer) !== self::CONFIRMATION) {
|
||||||
$output->writeln('<error>Not confirmed.</error>');
|
$output->writeln('<error>Not confirmed.</error>');
|
||||||
return 1;
|
return 1;
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\KeyService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
use Symfony\Component\Console\Question\Question;
|
||||||
|
|
||||||
|
final class KeyImport extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private KeyService $keys,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:key:import')
|
||||||
|
->setDescription('Disaster recovery: adopt the backup key from a recovery kit on a fresh installation')
|
||||||
|
->addArgument('kit-file', InputArgument::REQUIRED, 'The recovery kit file (occ backup:key:kit on the original installation)')
|
||||||
|
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking')
|
||||||
|
->setHelp(<<<'HELP'
|
||||||
|
For a fresh installation only (occ backup:key:init must not have been run here yet). Reads one
|
||||||
|
of the kit's passphrase slots to recover the original master key, so occ backup:target:add opens
|
||||||
|
the existing repository on the location instead of creating a new, empty one.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
occ backup:key:import /root/ng_backup-recovery-kit.json
|
||||||
|
occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=...
|
||||||
|
occ backup:restore:full offsite <snapshot>
|
||||||
|
HELP);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
if ($this->keys->isInitialized()) {
|
||||||
|
$output->writeln('<error>A backup key already exists here (fingerprint ' . $this->keys->fingerprint() . '). occ backup:key:import is only for a fresh installation.</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$kitFile = (string)$input->getArgument('kit-file');
|
||||||
|
$raw = @file_get_contents($kitFile);
|
||||||
|
if ($raw === false) {
|
||||||
|
$output->writeln("<error>Cannot read $kitFile</error>");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$kit = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
} catch (\JsonException $e) {
|
||||||
|
$output->writeln('<error>Not a valid recovery kit file: ' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$file = $input->getOption('passphrase-file');
|
||||||
|
if ($file !== null) {
|
||||||
|
$raw = @file_get_contents($file);
|
||||||
|
if ($raw === false) {
|
||||||
|
$output->writeln('<error>Cannot read the passphrase file</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$pass = rtrim($raw, "\r\n");
|
||||||
|
} else {
|
||||||
|
$helper = $this->getHelper('question');
|
||||||
|
$q = (new Question('Passphrase of one of the kit\'s slots: '))->setHidden(true)->setHiddenFallback(false);
|
||||||
|
$pass = (string)$helper->ask($input, $output, $q);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$this->keys->importRecoveryKit($kit, $pass);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$output->writeln('Backup key imported, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
||||||
|
// Read from the kit file, which is not authenticated beyond the wrapped key: show it so a
|
||||||
|
// wrong or altered value stands out.
|
||||||
|
$output->writeln('Deletion delay for forgotten snapshots: <info>' . $this->keys->deleteDelayDays() . '</info> days (from the kit).');
|
||||||
|
$output->writeln('Next: connect the backup location with the same details as before (<info>occ backup:target:add</info>), then <info>occ backup:list</info> and <info>occ backup:restore:full</info>.');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,7 +14,7 @@ use Symfony\Component\Console\Input\InputOption;
|
|||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
use Symfony\Component\Console\Question\Question;
|
use Symfony\Component\Console\Question\Question;
|
||||||
|
|
||||||
class KeyInit extends Command {
|
final class KeyInit extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
) {
|
) {
|
||||||
@@ -25,6 +25,8 @@ class KeyInit extends Command {
|
|||||||
$this->setName('backup:key:init')
|
$this->setName('backup:key:init')
|
||||||
->setDescription('Create the backup key, protected by a passphrase you keep safe')
|
->setDescription('Create the backup key, protected by a passphrase you keep safe')
|
||||||
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking')
|
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking')
|
||||||
|
->addOption('label', 'l', InputOption::VALUE_REQUIRED, 'Label of the first passphrase slot (who/where), e.g. "Safe"', 'Slot 1')
|
||||||
|
->addOption('delete-delay-days', null, InputOption::VALUE_REQUIRED, 'Days a removed snapshot stays in the trash before its data is deleted (1-365, fixed after this)', (string)KeyService::DEFAULT_DELETE_DELAY)
|
||||||
->setHelp(<<<'HELP'
|
->setHelp(<<<'HELP'
|
||||||
Creates the master key that encrypts all backups of this installation.
|
Creates the master key that encrypts all backups of this installation.
|
||||||
|
|
||||||
@@ -57,12 +59,14 @@ HELP);
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
$this->keys->initialize($pass);
|
$this->keys->initialize($pass, (int)$input->getOption('delete-delay-days'), (string)$input->getOption('label'));
|
||||||
} catch (\InvalidArgumentException $e) {
|
} catch (\InvalidArgumentException $e) {
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
$output->writeln('Backup key created, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
$output->writeln('Backup key created, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
||||||
|
$output->writeln('Removed snapshots stay in the trash on the location for <info>' . $this->keys->deleteDelayDays() . ' days</info> (cannot be changed later).');
|
||||||
|
$output->writeln('Tip: add up to two more passphrases for other people with occ backup:key:slot:add (e.g. CTO, head of IT).');
|
||||||
$output->writeln('Next: save the recovery kit with <info>occ backup:key:kit --output=FILE</info> and confirm with <info>occ backup:key:confirm</info>.');
|
$output->writeln('Next: save the recovery kit with <info>occ backup:key:kit --output=FILE</info> and confirm with <info>occ backup:key:confirm</info>.');
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
+18
-3
@@ -13,9 +13,10 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class KeyKit extends Command {
|
final class KeyKit extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
|
private \OCA\NgBackup\Service\AlertService $alerts,
|
||||||
) {
|
) {
|
||||||
parent::__construct();
|
parent::__construct();
|
||||||
}
|
}
|
||||||
@@ -31,17 +32,31 @@ class KeyKit extends Command {
|
|||||||
$output->writeln('<error>No backup key yet: run occ backup:key:init first.</error>');
|
$output->writeln('<error>No backup key yet: run occ backup:key:init first.</error>');
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
$this->alerts->securityEvent('kit_downloaded');
|
||||||
$json = json_encode($this->keys->recoveryKit(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n";
|
$json = json_encode($this->keys->recoveryKit(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n";
|
||||||
$file = $input->getOption('output');
|
$file = $input->getOption('output');
|
||||||
if ($file === null) {
|
if ($file === null) {
|
||||||
$output->write($json, false, OutputInterface::OUTPUT_RAW);
|
$output->write($json, false, OutputInterface::OUTPUT_RAW);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
if (file_put_contents($file, $json) === false) {
|
// Created owner-only from the start (no window where it is world-readable), never over an
|
||||||
|
// existing file or symlink someone else may have placed at a predictable path.
|
||||||
|
$old = umask(0077);
|
||||||
|
try {
|
||||||
|
$fh = @fopen($file, 'xb');
|
||||||
|
} finally {
|
||||||
|
umask($old);
|
||||||
|
}
|
||||||
|
if ($fh === false) {
|
||||||
|
$output->writeln("<error>Cannot create $file (it must not exist yet)</error>");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$ok = fwrite($fh, $json) === strlen($json);
|
||||||
|
if (!fclose($fh) || !$ok) {
|
||||||
|
@unlink($file);
|
||||||
$output->writeln("<error>Cannot write $file</error>");
|
$output->writeln("<error>Cannot write $file</error>");
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
@chmod($file, 0600);
|
|
||||||
$output->writeln("Recovery kit written to <info>$file</info>. Copy it off this server, then run <info>occ backup:key:confirm</info>.");
|
$output->writeln("Recovery kit written to <info>$file</info>. Copy it off this server, then run <info>occ backup:key:confirm</info>.");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\KeyService;
|
||||||
|
use OCA\NgBackup\Service\KeySlotService;
|
||||||
|
|
||||||
|
final class KeySlotAdd extends KeySlots {
|
||||||
|
public function __construct(KeyService $keys, KeySlotService $slots) {
|
||||||
|
parent::__construct($keys, $slots, 'add');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\KeyService;
|
||||||
|
use OCA\NgBackup\Service\KeySlotService;
|
||||||
|
|
||||||
|
final class KeySlotRemove extends KeySlots {
|
||||||
|
public function __construct(KeyService $keys, KeySlotService $slots) {
|
||||||
|
parent::__construct($keys, $slots, 'remove');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\KeyService;
|
||||||
|
use OCA\NgBackup\Service\KeySlotService;
|
||||||
|
|
||||||
|
final class KeySlotReplace extends KeySlots {
|
||||||
|
public function __construct(KeyService $keys, KeySlotService $slots) {
|
||||||
|
parent::__construct($keys, $slots, 'replace');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\KeyService;
|
||||||
|
use OCA\NgBackup\Service\KeySlotService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Helper\Table;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
use Symfony\Component\Console\Question\Question;
|
||||||
|
|
||||||
|
/** backup:key:slots, backup:key:slot:add, backup:key:slot:replace, backup:key:slot:remove */
|
||||||
|
class KeySlots extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private KeyService $keys,
|
||||||
|
private KeySlotService $slots,
|
||||||
|
private string $action = 'list',
|
||||||
|
) {
|
||||||
|
// list is the default; subclasses pick another action
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$help = <<<'HELP'
|
||||||
|
Up to three passphrases ("slots") can open the backup key, so that one person leaving or dying
|
||||||
|
does not lock you out. Example: one passphrase in the safe, one with the CTO, one with the head
|
||||||
|
of IT. Each holder keeps a copy of the same recovery kit and only knows their own passphrase.
|
||||||
|
After every change: occ backup:key:kit and occ backup:key:confirm.
|
||||||
|
HELP;
|
||||||
|
match ($this->action) {
|
||||||
|
'add' => $this->setName('backup:key:slot:add')->setDescription('Add a passphrase slot')
|
||||||
|
->addOption('label', 'l', InputOption::VALUE_REQUIRED, 'Who or where, e.g. "CTO" or "Safe"', 'Slot'),
|
||||||
|
'replace' => $this->setName('backup:key:slot:replace')->setDescription('Set a new passphrase for a slot (e.g. its holder left)')
|
||||||
|
->addArgument('slot', InputArgument::REQUIRED, 'Slot number (see backup:key:slots)')
|
||||||
|
->addOption('label', 'l', InputOption::VALUE_REQUIRED, 'New label'),
|
||||||
|
'remove' => $this->setName('backup:key:slot:remove')->setDescription('Remove a passphrase slot (not the last one)')
|
||||||
|
->addArgument('slot', InputArgument::REQUIRED, 'Slot number (see backup:key:slots)'),
|
||||||
|
default => $this->setName('backup:key:slots')->setDescription('List the passphrase slots of the backup key'),
|
||||||
|
};
|
||||||
|
if ($this->action === 'add' || $this->action === 'replace') {
|
||||||
|
$this->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file');
|
||||||
|
}
|
||||||
|
$this->setHelp($help);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
if (!$this->keys->isInitialized()) {
|
||||||
|
$output->writeln('<error>No backup key yet: run occ backup:key:init first.</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
switch ($this->action) {
|
||||||
|
case 'add':
|
||||||
|
$n = $this->slots->add($this->passphrase($input, $output), (string)$input->getOption('label'));
|
||||||
|
$output->writeln("Slot $n added.");
|
||||||
|
break;
|
||||||
|
case 'replace':
|
||||||
|
$this->slots->replace((int)$input->getArgument('slot'), $this->passphrase($input, $output), $input->getOption('label'));
|
||||||
|
$output->writeln('Slot ' . (int)$input->getArgument('slot') . ' has a new passphrase.');
|
||||||
|
break;
|
||||||
|
case 'remove':
|
||||||
|
$this->slots->remove((int)$input->getArgument('slot'));
|
||||||
|
$output->writeln('Slot ' . (int)$input->getArgument('slot') . ' removed.');
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
(new Table($output))->setHeaders(['slot', 'label', 'set on'])
|
||||||
|
->setRows(array_map(fn ($s) => [$s['slot'], $s['label'], $s['created'] ? date('Y-m-d H:i', $s['created']) : ''], $this->keys->slots()))->render();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$output->writeln('Now download and confirm the new recovery kit: occ backup:key:kit -o FILE && occ backup:key:confirm');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function passphrase(InputInterface $input, OutputInterface $output): string {
|
||||||
|
if ($file = $input->getOption('passphrase-file')) {
|
||||||
|
return rtrim((string)@file_get_contents($file), "\r\n");
|
||||||
|
}
|
||||||
|
$h = $this->getHelper('question');
|
||||||
|
$p = (string)$h->ask($input, $output, (new Question('Passphrase (min. 12 characters): '))->setHidden(true)->setHiddenFallback(false));
|
||||||
|
if ($p !== (string)$h->ask($input, $output, (new Question('Repeat: '))->setHidden(true)->setHiddenFallback(false))) {
|
||||||
|
throw new \InvalidArgumentException('The passphrases do not match');
|
||||||
|
}
|
||||||
|
return $p;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputArgument;
|
|||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class ListSnapshots extends Command {
|
final class ListSnapshots extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private SnapshotMapper $snapshots,
|
private SnapshotMapper $snapshots,
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\PruneService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class Prune extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private PruneService $prune,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:prune')
|
||||||
|
->setDescription('Apply the retention policy: remove old snapshots and the data only they used')
|
||||||
|
->addArgument('target', InputArgument::OPTIONAL, 'Location name or id (default: all that are not append-only)')
|
||||||
|
->addOption('dry-run', null, InputOption::VALUE_NONE, 'Only show what would be removed');
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$name = $input->getArgument('target');
|
||||||
|
$targets = $name === null ? array_filter($this->targets->list(), fn ($t) => !$t->getAppendOnly()) : [$this->targets->get($name)];
|
||||||
|
$dry = (bool)$input->getOption('dry-run');
|
||||||
|
$output->writeln('Policy: ' . json_encode($this->prune->policy()->toArray()) . ($dry ? ' (dry run)' : ''));
|
||||||
|
$rc = 0;
|
||||||
|
foreach ($targets as $t) {
|
||||||
|
// User exports first: Repository::prune()'s blob garbage collection (run as part of
|
||||||
|
// apply() below) scans the users/ manifests still on the location at that point, so
|
||||||
|
// forgetting one here lets its now-unused blobs be freed in this same pass.
|
||||||
|
try {
|
||||||
|
$ru = $this->prune->applyUserExports($t, $dry);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
// Snapshot retention is independent of this: report and carry on with it.
|
||||||
|
$output->writeln('<error>' . $t->getName() . ' (user exports): ' . $e->getMessage() . '</error>');
|
||||||
|
$rc = 1;
|
||||||
|
$ru = ['forgotten' => []];
|
||||||
|
}
|
||||||
|
if ($ru['forgotten'] !== []) {
|
||||||
|
$affected = count(array_unique(array_map(static fn (string $path): string => explode('/', $path)[1] ?? '', $ru['forgotten'])));
|
||||||
|
$output->writeln(sprintf('<info>%s</info>: %s %d user export(s) for %d user(s)',
|
||||||
|
$t->getName(), $dry ? 'would forget' : 'forgot', count($ru['forgotten']), $affected));
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$r = $this->prune->apply($t, $dry);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $t->getName() . ': ' . $e->getMessage() . '</error>');
|
||||||
|
$rc = 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$p = $r['prune'];
|
||||||
|
$output->writeln(sprintf('<info>%s</info>: keep %d, %s %d snapshots; packs: %d, %s %d, repack %d, frees %.1f MiB',
|
||||||
|
$t->getName(), count($r['kept']), $dry ? 'would forget' : 'forgot', count($r['forgotten']), $p['packs'],
|
||||||
|
$dry ? 'would delete' : 'deleted', $p['deleted'], $p['repacked'], $p['freedBytes'] / 1048576));
|
||||||
|
}
|
||||||
|
return $rc;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class Restore extends Command {
|
final class Restore extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RestoreService $restore,
|
private RestoreService $restore,
|
||||||
@@ -49,7 +49,7 @@ HELP);
|
|||||||
try {
|
try {
|
||||||
$target = $this->targets->get($input->getArgument('target'));
|
$target = $this->targets->get($input->getArgument('target'));
|
||||||
$snapshot = $input->getArgument('snapshot');
|
$snapshot = $input->getArgument('snapshot');
|
||||||
$path = $input->getArgument('path');
|
$path = trim((string)$input->getArgument('path'), '/'); // CLI convenience; the service itself only accepts canonical paths
|
||||||
if ($dir = $input->getOption('to-directory')) {
|
if ($dir = $input->getOption('to-directory')) {
|
||||||
$n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir);
|
$n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir);
|
||||||
$output->writeln("Restored <info>$n</info> files to $dir.");
|
$output->writeln("Restored <info>$n</info> files to $dir.");
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\RestoreFullService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCP\IUserManager;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Helper\QuestionHelper;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class RestoreFull extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private RestoreFullService $restoreFull,
|
||||||
|
private IUserManager $userManager,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:restore:full')
|
||||||
|
->setDescription('Disaster recovery: restore the database, data directory and config on a fresh installation')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
||||||
|
->addOption('force', null, InputOption::VALUE_NONE, 'Also on an installation that already has more than one user, and without asking')
|
||||||
|
->setHelp(<<<'HELP'
|
||||||
|
For a FRESH Nextcloud installation only, same major version as the backup, right after
|
||||||
|
occ maintenance:install, occ backup:key:import and occ backup:target:add -- before anyone else
|
||||||
|
uses this instance.
|
||||||
|
|
||||||
|
Every app that was enabled when the snapshot was taken must already be installed here (its
|
||||||
|
database tables must exist before the restore can write into them); the command lists what is
|
||||||
|
missing and refuses to continue rather than guess.
|
||||||
|
|
||||||
|
Steps: the database is replaced (consistent, in one transaction); the data directory is restored
|
||||||
|
directly in place; config.php is merged in -- everything from the backup except the database
|
||||||
|
connection and data directory settings, which stay as occ maintenance:install configured them
|
||||||
|
here. Other files under config/ (if any) are not restored.
|
||||||
|
|
||||||
|
Afterwards: check trusted_domains and overwritehost/overwriteprotocol match this server, and
|
||||||
|
occ maintenance:repair.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
occ backup:restore:full offsite 537bf7e9c1a4...
|
||||||
|
HELP);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$force = (bool)$input->getOption('force');
|
||||||
|
// A whole-server restore replaces the database and data directory of THIS instance. On
|
||||||
|
// anything but a fresh install (only the admin from occ maintenance:install) that is
|
||||||
|
// almost certainly a mistake, so it needs --force.
|
||||||
|
$users = $this->userManager->countUsersTotal(2);
|
||||||
|
if (!$force && ($users === false || $users > 1)) {
|
||||||
|
$output->writeln('<error>This installation already has more than one user: backup:restore:full would replace its whole database and data directory. It is meant for a fresh installation. Use --force if this really is what you want.</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (!$force && $input->isInteractive()) {
|
||||||
|
$q = new ConfirmationQuestion('This replaces the database, the data directory and config.php of this installation. Continue? [y/N] ', false);
|
||||||
|
if (!(new QuestionHelper())->ask($input, $output, $q)) {
|
||||||
|
$output->writeln('Cancelled.');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$target = $this->targets->get($input->getArgument('target'));
|
||||||
|
$snapshot = (string)$input->getArgument('snapshot');
|
||||||
|
$result = $this->restoreFull->restoreFull($target, $snapshot);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
$output->writeln('If this happened after the restore had started, the instance may be partly restored: fix the cause and run the same command again with --force.');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$output->writeln(sprintf(
|
||||||
|
'Database restored (<info>%d</info> tables, <info>%d</info> unchanged), config merged (<info>%d</info> keys), <info>%d</info> data files restored.',
|
||||||
|
count($result['dbReport']['restored']),
|
||||||
|
count($result['dbReport']['skipped']),
|
||||||
|
$result['configKeys'],
|
||||||
|
$result['dataFiles'],
|
||||||
|
));
|
||||||
|
if (!$result['dbConsistent']) {
|
||||||
|
$output->writeln('<comment>Warning: this snapshot\'s database dump was made over several steps (web cron), so it is not one consistent point in time. Check the instance carefully; prefer a snapshot made with system cron.</comment>');
|
||||||
|
}
|
||||||
|
$output->writeln('Next: check trusted_domains and overwritehost/overwriteprotocol match this server, then occ maintenance:repair.');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCA\NgBackup\Service\UserRestoreService;
|
||||||
|
use OCA\NgBackup\UserMigration\UserMigrationInstaller;
|
||||||
|
use OCP\IUserManager;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Helper\QuestionHelper;
|
||||||
|
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class RestoreUser extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private UserRestoreService $userRestore,
|
||||||
|
private UserMigrationInstaller $installer,
|
||||||
|
private IUserManager $userManager,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:user:restore')
|
||||||
|
->setDescription('Restore a user from a user_migration export (see backup:user:backup)')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addArgument('manifest', InputArgument::REQUIRED, 'Manifest path printed by backup:user:backup (users/<uid>/<exportId>)')
|
||||||
|
->addOption('mode', 'm', InputOption::VALUE_REQUIRED, 'replace or as-backup', UserRestoreService::MODE_REPLACE)
|
||||||
|
->addOption('force', null, InputOption::VALUE_NONE, 'Replace an existing account without asking (required when not interactive)')
|
||||||
|
->addOption('install-user-migration', null, InputOption::VALUE_NONE, 'Install and enable user_migration if it is missing (non-interactive consent)')
|
||||||
|
->setHelp(<<<'HELP'
|
||||||
|
Never imports over a live account. Modes:
|
||||||
|
replace (default) safety-exports the current account first, then deletes and recreates it
|
||||||
|
fresh from the backup, under the same user id
|
||||||
|
as-backup restores into a new account "<uid>-bak" (or "-bak-2", ...) instead, so an admin can
|
||||||
|
compare and transfer by hand; Nextcloud cannot rename a user id, so the backup gets
|
||||||
|
the new name, never the existing account
|
||||||
|
|
||||||
|
Example:
|
||||||
|
occ backup:user:restore offsite users/alice/20261005T120000Z-ab12cd34
|
||||||
|
occ backup:user:restore offsite users/alice/20261005T120000Z-ab12cd34 --mode=as-backup
|
||||||
|
HELP);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$mode = (string)$input->getOption('mode');
|
||||||
|
$uid = explode('/', (string)$input->getArgument('manifest'))[1] ?? '';
|
||||||
|
if ($mode === UserRestoreService::MODE_REPLACE && $uid !== '' && $this->userManager->userExists($uid) && !$input->getOption('force')) {
|
||||||
|
// Replace deletes the live account (after a safety export). Shares to others and data of
|
||||||
|
// apps without a user_migration migrator do not come back, so never do it unasked.
|
||||||
|
if (!$input->isInteractive()) {
|
||||||
|
$output->writeln("<error>$uid exists: --mode=replace deletes and recreates it. Add --force, or use --mode=as-backup.</error>");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$q = new ConfirmationQuestion("$uid exists and will be deleted and recreated from the backup (a safety export is made first; its shares to others and data of apps without a migrator are not restored). Continue? [y/N] ", false);
|
||||||
|
if (!(new QuestionHelper())->ask($input, $output, $q)) {
|
||||||
|
$output->writeln('Cancelled.');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$output->writeln('<comment>Note: on restore, user_migration only imports the app settings on its own allowlist (e.g. calendar view and reminder settings); other app settings of this user are skipped. Reason: NG Backup uses user_migration\'s importer; widening the list would mean building a settings importer of our own.</comment>');
|
||||||
|
$target = $this->targets->get($input->getArgument('target'));
|
||||||
|
$this->installer->ensureEnabled($input, $output);
|
||||||
|
$user = $this->userRestore->restoreUser($target, (string)$input->getArgument('manifest'), $mode);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$output->writeln('Restored <info>' . $user->getUID() . '</info>.');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\PruneService;
|
||||||
|
use OCA\NgBackup\Service\RetentionPolicy;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class Retention extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private PruneService $prune,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:retention')
|
||||||
|
->setDescription('Show or set how many snapshots are kept')
|
||||||
|
->addOption('last', null, InputOption::VALUE_REQUIRED, 'Always keep the newest N')
|
||||||
|
->addOption('daily', null, InputOption::VALUE_REQUIRED, 'Keep one per day for the last N days with a backup')
|
||||||
|
->addOption('weekly', null, InputOption::VALUE_REQUIRED, 'Keep one per week for N weeks')
|
||||||
|
->addOption('monthly', null, InputOption::VALUE_REQUIRED, 'Keep one per month for N months')
|
||||||
|
->addOption('user-last', null, InputOption::VALUE_REQUIRED, 'Keep the newest N of each user\'s own exports (occ backup:user:backup)')
|
||||||
|
->setHelp("Example: occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12 --user-last=3");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
if ($input->getOption('user-last') !== null && !ctype_digit((string)$input->getOption('user-last'))) {
|
||||||
|
$output->writeln('<error>--user-last must be a whole number</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$p = $this->prune->policy()->toArray();
|
||||||
|
$changed = false;
|
||||||
|
foreach (['last', 'daily', 'weekly', 'monthly'] as $k) {
|
||||||
|
if ($input->getOption($k) !== null) {
|
||||||
|
$p[$k] = max(0, (int)$input->getOption($k));
|
||||||
|
$changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($changed) {
|
||||||
|
try {
|
||||||
|
$this->prune->setPolicy(RetentionPolicy::fromArray($p));
|
||||||
|
} catch (\InvalidArgumentException $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($input->getOption('user-last') !== null) {
|
||||||
|
try {
|
||||||
|
$this->prune->setUserExportRetention((int)$input->getOption('user-last'));
|
||||||
|
} catch (\InvalidArgumentException $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$output->writeln(sprintf('Keep the last %d, plus one per day for %d days, per week for %d weeks, per month for %d months.', $p['last'], $p['daily'], $p['weekly'], $p['monthly']));
|
||||||
|
$output->writeln(sprintf('Keep the newest %d of each user\'s own exports.', $this->prune->userExportRetention()));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,7 +16,7 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class RunBackup extends Command {
|
final class RunBackup extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private BackupService $backups,
|
private BackupService $backups,
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
@@ -75,6 +75,13 @@ HELP);
|
|||||||
});
|
});
|
||||||
if ($run->getStatus() === Run::DONE) {
|
if ($run->getStatus() === Run::DONE) {
|
||||||
$output->writeln(sprintf(' done in %.0fs', microtime(true) - $start));
|
$output->writeln(sprintf(' done in %.0fs', microtime(true) - $start));
|
||||||
|
$f = json_decode($run->getStats() ?? '{}', true)['files'] ?? [];
|
||||||
|
if (($f['unreadable'] ?? 0) > 0) {
|
||||||
|
$output->writeln(sprintf('<comment> warning: %d unreadable files or folders are missing from this backup (see the server log)</comment>', $f['unreadable']));
|
||||||
|
}
|
||||||
|
if (($f['changing'] ?? 0) > 0) {
|
||||||
|
$output->writeln(sprintf('<comment> warning: %d files kept changing and were saved as read (see the server log)</comment>', $f['changing']));
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
$output->writeln('<error> ' . $run->getStatus() . ': ' . $run->getError() . '</error>');
|
$output->writeln('<error> ' . $run->getStatus() . ': ' . $run->getError() . '</error>');
|
||||||
$failed++;
|
$failed++;
|
||||||
|
|||||||
+14
-2
@@ -10,16 +10,18 @@ namespace OCA\NgBackup\Command;
|
|||||||
use OCA\NgBackup\Db\RunMapper;
|
use OCA\NgBackup\Db\RunMapper;
|
||||||
use OCA\NgBackup\Service\KeyService;
|
use OCA\NgBackup\Service\KeyService;
|
||||||
use OCA\NgBackup\Service\TargetService;
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCA\NgBackup\Service\VerifyService;
|
||||||
use Symfony\Component\Console\Command\Command;
|
use Symfony\Component\Console\Command\Command;
|
||||||
use Symfony\Component\Console\Helper\Table;
|
use Symfony\Component\Console\Helper\Table;
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class Status extends Command {
|
final class Status extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RunMapper $runs,
|
private RunMapper $runs,
|
||||||
|
private VerifyService $verify,
|
||||||
) {
|
) {
|
||||||
parent::__construct();
|
parent::__construct();
|
||||||
}
|
}
|
||||||
@@ -44,13 +46,23 @@ class Status extends Command {
|
|||||||
$rows = [];
|
$rows = [];
|
||||||
foreach ($this->runs->findRecent(10) as $r) {
|
foreach ($this->runs->findRecent(10) as $r) {
|
||||||
$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
|
$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
|
||||||
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . ')' : ''),
|
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . self::percent($r) . ')' : ''),
|
||||||
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? gmdate('H:i:s', $r->getFinishedAt() - $r->getStartedAt()) : '',
|
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? gmdate('H:i:s', $r->getFinishedAt() - $r->getStartedAt()) : '',
|
||||||
$s['files'] ?? '', isset($s['uploaded']) ? sprintf('%.1f MiB', $s['uploaded'] / 1048576) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
|
$s['files'] ?? '', isset($s['uploaded']) ? sprintf('%.1f MiB', $s['uploaded'] / 1048576) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
|
||||||
}
|
}
|
||||||
if ($rows !== []) {
|
if ($rows !== []) {
|
||||||
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'uploaded', 'error'])->setRows($rows)->render();
|
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'uploaded', 'error'])->setRows($rows)->render();
|
||||||
}
|
}
|
||||||
|
foreach ($this->verify->progress() as $targetId => $state) {
|
||||||
|
$output->writeln(sprintf('Verify of %s on %s: %s', mb_strimwidth((string)($state['what'] ?? ''), 0, 40, '…'),
|
||||||
|
$names[$targetId] ?? '#' . $targetId, VerifyService::describe($state)));
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** " 68%", or " ~68%" when estimated from the file cache; empty without an estimate */
|
||||||
|
private static function percent(\OCA\NgBackup\Db\Run $r): string {
|
||||||
|
$p = \OCA\NgBackup\Job\BackupRun::progressOf($r);
|
||||||
|
return $p === null ? '' : sprintf(' %s%d%%', $p['approximate'] ? '~' : '', (int)floor($p['fraction'] * 100.0));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class TargetAdd extends Command {
|
final class TargetAdd extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class TargetAppendOnly extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private \OCA\NgBackup\Service\AlertService $alerts,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:target:append-only')
|
||||||
|
->setDescription('Make a location append-only (NG Backup never deletes there) or normal again')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addArgument('state', InputArgument::REQUIRED, 'on or off')
|
||||||
|
->setHelp("Append-only protects backups if this server is compromised: combine it with credentials\nwithout delete rights and a lifecycle rule or object lock on the location itself.");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$state = strtolower((string)$input->getArgument('state'));
|
||||||
|
if (!in_array($state, ['on', 'off'], true)) {
|
||||||
|
$output->writeln('<error>state must be on or off</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$t = $this->targets->setAppendOnly($this->targets->get($input->getArgument('target')), $state === 'on');
|
||||||
|
if ($state === 'off') {
|
||||||
|
$this->alerts->securityEvent('append_only_off', ['target' => $t->getName()]);
|
||||||
|
}
|
||||||
|
$output->writeln($t->getName() . ' is ' . ($t->getAppendOnly() ? 'append-only' : 'normal') . '.');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,7 +12,7 @@ use OCA\NgBackup\Service\TargetService;
|
|||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class TargetBackends extends Base {
|
final class TargetBackends extends Base {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
) {
|
) {
|
||||||
@@ -32,10 +32,16 @@ class TargetBackends extends Base {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
foreach ($backends as $b) {
|
foreach ($backends as $b) {
|
||||||
$output->writeln(sprintf('<info>%s</info> (%s) auth schemes: %s', $b['id'], $b['name'], implode(', ', $b['auth'])));
|
$output->writeln(sprintf('<info>%s</info> (%s)', $b['id'], $b['name']));
|
||||||
foreach ($b['parameters'] as $key => $p) {
|
foreach ($b['parameters'] as $key => $p) {
|
||||||
$output->writeln(sprintf(' -o %s=... %s', $key, is_array($p) ? ($p['value'] ?? '') : ''));
|
$output->writeln(sprintf(' -o %s=... %s', $key, is_array($p) ? ($p['value'] ?? '') : ''));
|
||||||
}
|
}
|
||||||
|
foreach ($b['auth'] as $a) {
|
||||||
|
$output->writeln(sprintf(' -a %s (%s)', $a['id'], $a['name']));
|
||||||
|
foreach ($a['parameters'] as $key => $p) {
|
||||||
|
$output->writeln(sprintf(' -o %s=... %s', $key, is_array($p) ? ($p['value'] ?? '') : ''));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\AlertService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class TargetEdit extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private AlertService $alerts,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:target:edit')
|
||||||
|
->setDescription('Change the connection settings or the name of a backup location, keeping its backups')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addOption('name', null, InputOption::VALUE_REQUIRED, 'New name')
|
||||||
|
->addOption('backend', 'b', InputOption::VALUE_REQUIRED, 'New files_external backend id (see backup:target:backends)')
|
||||||
|
->addOption('auth', 'a', InputOption::VALUE_REQUIRED, 'New auth mechanism id')
|
||||||
|
->addOption('option', 'o', InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Option key=value to change (repeatable); other options keep their value')
|
||||||
|
->addOption('path', 'p', InputOption::VALUE_REQUIRED, 'New folder on the location')
|
||||||
|
->setHelp(<<<'HELP'
|
||||||
|
The new settings must lead to the same repository as before, so the history of the location is
|
||||||
|
kept. Use it after a password change, a new host name, or after moving the repository.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
occ backup:target:edit nas -o password=new-secret
|
||||||
|
HELP);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
try {
|
||||||
|
$t = $this->targets->get($input->getArgument('target'));
|
||||||
|
$options = $this->targets->options($t);
|
||||||
|
if ($input->getOption('backend') !== null || $input->getOption('auth') !== null) {
|
||||||
|
$options = []; // a different connection type has its own options
|
||||||
|
}
|
||||||
|
foreach ($input->getOption('option') as $pair) {
|
||||||
|
if (!str_contains($pair, '=')) {
|
||||||
|
$output->writeln("<error>Option '$pair' must be key=value</error>");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$k = (string)strstr($pair, '=', true);
|
||||||
|
$v = substr($pair, strlen($k) + 1);
|
||||||
|
$options[$k] = match (strtolower($v)) { 'true' => true, 'false' => false, default => $v };
|
||||||
|
}
|
||||||
|
$before = $t->getName();
|
||||||
|
$t = $this->targets->update($t, $input->getOption('name') ?? $t->getName(), $input->getOption('backend') ?? $t->getBackend(),
|
||||||
|
$input->getOption('auth') ?? $t->getAuth(), $options, $input->getOption('path') ?? $t->getBasePath());
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$this->alerts->securityEvent('target_changed', ['target' => $before]);
|
||||||
|
$output->writeln(sprintf('Location <info>%s</info> updated; repository %s found with the new settings.', $t->getName(), (string)$t->getRepositoryId()));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class TargetLimit extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:target:limit')
|
||||||
|
->setDescription('Set a maximum size for the backups stored on a location (e.g. 100G), or "none"')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addArgument('size', InputArgument::OPTIONAL, 'Size like 500M, 20G, 2T, or "none"; omit to show the current limit')
|
||||||
|
->setHelp("A backup that would exceed the limit stops with an error and administrators are notified.\n"
|
||||||
|
. "Note: with versioning or Object Lock on the location, removed data keeps using space there\n"
|
||||||
|
. "until it expires; keep the limit below what you are willing to pay for.");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$t = $this->targets->get($input->getArgument('target'));
|
||||||
|
$size = $input->getArgument('size');
|
||||||
|
if ($size !== null) {
|
||||||
|
if (strtolower($size) === 'none') {
|
||||||
|
$t = $this->targets->setMaxBytes($t, null);
|
||||||
|
} elseif (preg_match('/^(\d+(?:\.\d+)?)\s*([KMGT]?)i?B?$/i', $size, $m)) {
|
||||||
|
$factor = ['' => 1, 'K' => 1024, 'M' => 1024 ** 2, 'G' => 1024 ** 3, 'T' => 1024 ** 4][strtoupper($m[2])];
|
||||||
|
$t = $this->targets->setMaxBytes($t, (int)round((float)$m[1] * $factor));
|
||||||
|
} else {
|
||||||
|
$output->writeln('<error>Use a size like 500M, 20G or 2T, or "none".</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$used = $this->targets->repository($t)->storedBytes();
|
||||||
|
$output->writeln(sprintf('%s: %.2f GiB stored, limit %s', $t->getName(), $used / 1073741824,
|
||||||
|
$t->getMaxBytes() === null ? 'none' : sprintf('%.2f GiB', $t->getMaxBytes() / 1073741824)));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,7 +13,7 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class TargetList extends Base {
|
final class TargetList extends Base {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -13,9 +13,10 @@ use Symfony\Component\Console\Input\InputArgument;
|
|||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
class TargetRemove extends Command {
|
final class TargetRemove extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
|
private \OCA\NgBackup\Service\AlertService $alerts,
|
||||||
) {
|
) {
|
||||||
parent::__construct();
|
parent::__construct();
|
||||||
}
|
}
|
||||||
@@ -34,6 +35,7 @@ class TargetRemove extends Command {
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
$this->targets->remove($t);
|
$this->targets->remove($t);
|
||||||
|
$this->alerts->securityEvent('target_removed', ['target' => $t->getName()]);
|
||||||
$output->writeln('Removed <info>' . $t->getName() . '</info>. The backups on the location itself were not touched; add it again to use them.');
|
$output->writeln('Removed <info>' . $t->getName() . '</info>. The backups on the location itself were not touched; add it again to use them.');
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Db\AppConfigCatalogAnchor;
|
||||||
|
use OCA\NgBackup\Service\AlertService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCP\IAppConfig;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
use Symfony\Component\Console\Question\Question;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Accept the state a location shows now, after NG Backup refused it as rolled back. Only for
|
||||||
|
* legitimate cases (e.g. the location was restored from the provider's own snapshot), only from
|
||||||
|
* occ, audited and announced to all administrators.
|
||||||
|
*/
|
||||||
|
final class TargetTrustCurrent extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private IAppConfig $appConfig,
|
||||||
|
private AlertService $alerts,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:target:trust-current')
|
||||||
|
->setDescription('Accept the current state of a location that was refused as rolled back')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->setHelp("NG Backup refuses a location whose backup catalog is older than what this server saw\n"
|
||||||
|
. "before, because an attacker could have restored an old state. If you restored the location\n"
|
||||||
|
. "yourself (e.g. from the provider's snapshot), accept it with this command.");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$t = $this->targets->get($input->getArgument('target'));
|
||||||
|
$output->writeln('<comment>Only continue if you know why the location shows an older state.</comment>');
|
||||||
|
$answer = (string)$this->getHelper('question')->ask($input, $output, new Question('Type the location name to accept its current state: '));
|
||||||
|
if ($answer !== $t->getName()) {
|
||||||
|
$output->writeln('<error>Not confirmed.</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$this->appConfig->deleteKey('ng_backup', 'catalog_' . preg_replace('/[^a-f0-9]/', '', (string)$t->getRepositoryId()));
|
||||||
|
$repo = $this->targets->repository($t); // re-anchors on the newest valid catalog
|
||||||
|
$this->alerts->securityEvent('catalog_trusted', ['target' => $t->getName(), 'generation' => (string)$repo->catalogGeneration()]);
|
||||||
|
$output->writeln(sprintf('Accepted generation %d of %s (%d snapshots).', $repo->catalogGeneration(), $t->getName(), count($repo->listSnapshots())));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\KeyService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Helper\Table;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class Trash extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private KeyService $keys,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:trash')
|
||||||
|
->setDescription('List removed snapshots that can still be restored (backup:trash:restore)')
|
||||||
|
->addArgument('target', InputArgument::OPTIONAL, 'Location name or id (default: all)');
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
$name = $input->getArgument('target');
|
||||||
|
$delay = $this->keys->deleteDelayDays() * 86400;
|
||||||
|
$rows = [];
|
||||||
|
foreach ($name === null ? $this->targets->list() : [$this->targets->get($name)] as $t) {
|
||||||
|
foreach ($this->targets->repository($t)->trash() as $id => $info) {
|
||||||
|
$rows[] = [$t->getName(), $id, (string)($info['time'] ?? ''), $info['label'], date('Y-m-d H:i', $info['forgottenAt']), $info['by'],
|
||||||
|
date('Y-m-d H:i', $info['forgottenAt'] + $delay)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($rows === []) {
|
||||||
|
$output->writeln('The trash is empty.');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
(new Table($output))->setHeaders(['location', 'snapshot', 'made', 'label', 'removed', 'by', 'deleted after'])->setRows($rows)->render();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\PruneService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class TrashRestore extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private PruneService $prune,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:trash:restore')
|
||||||
|
->setDescription('Bring a removed snapshot back from the trash')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:trash)');
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
try {
|
||||||
|
$this->prune->untrash($this->targets->get($input->getArgument('target')), $input->getArgument('snapshot'));
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$output->writeln('Snapshot restored from the trash.');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Command;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCA\NgBackup\Service\VerifyService;
|
||||||
|
use Symfony\Component\Console\Command\Command;
|
||||||
|
use Symfony\Component\Console\Input\InputArgument;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
|
final class Verify extends Command {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private VerifyService $verify,
|
||||||
|
) {
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configure(): void {
|
||||||
|
$this->setName('backup:verify')
|
||||||
|
->setDescription('Checksum audit of a snapshot: every blob it needs must be in the (freshly re-checked) index and its pack present')
|
||||||
|
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||||
|
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
||||||
|
->addOption('deep', null, InputOption::VALUE_NONE, 'Also download and decrypt every blob (verifies its authentication tag and content hash, not just that it is listed); much slower')
|
||||||
|
->setHelp(<<<'HELP'
|
||||||
|
Without --deep this re-downloads and re-authenticates the index (never trusting the local cache)
|
||||||
|
and confirms every blob the snapshot needs is listed and its pack is present -- fast, no blob
|
||||||
|
content is read, so it cannot by itself catch a truncated or bit-rotted pack. With --deep every
|
||||||
|
blob is also downloaded and decrypted, which fails loudly on exactly that even if the index still
|
||||||
|
lists it -- the same verification an actual restore would do, run here deliberately instead of
|
||||||
|
waiting to find out during one.
|
||||||
|
|
||||||
|
While it runs it prints how far it is every 30 seconds; occ backup:status shows the same, and
|
||||||
|
how the last check of each location ended.
|
||||||
|
|
||||||
|
Run this periodically (cron, picking a sample of recent snapshots) rather than only right before
|
||||||
|
you'd need to rely on a restore.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
occ backup:verify offsite 537bf7e9c1a4...
|
||||||
|
occ backup:verify offsite 537bf7e9c1a4... --deep
|
||||||
|
HELP);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
|
try {
|
||||||
|
$target = $this->targets->get($input->getArgument('target'));
|
||||||
|
$ref = (string)$input->getArgument('snapshot');
|
||||||
|
$deep = (bool)$input->getOption('deep');
|
||||||
|
$onProgress = static function (array $state) use ($output): void {
|
||||||
|
$output->writeln(date('H:i:s') . ' ' . VerifyService::describe($state));
|
||||||
|
};
|
||||||
|
$result = str_starts_with($ref, 'users/')
|
||||||
|
? $this->verify->verifyUserExport($target, $ref, $deep, $onProgress)
|
||||||
|
: $this->verify->verify($target, $ref, $deep, $onProgress);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
$output->writeln(sprintf('%d files, %d blobs checked%s.', $result['filesChecked'], $result['blobsChecked'],
|
||||||
|
$deep ? sprintf(' (%.1f MiB decrypted)', $result['bytesChecked'] / 1048576) : ''));
|
||||||
|
if ($result['missing'] !== []) {
|
||||||
|
$output->writeln('<error>' . count($result['missing']) . ' blob(s) missing from the index:</error> ' . implode(', ', $result['missing']));
|
||||||
|
}
|
||||||
|
if ($result['failed'] !== []) {
|
||||||
|
$output->writeln('<error>' . count($result['failed']) . ' blob(s) failed verification (corrupted or tampered):</error> ' . implode(', ', $result['failed']));
|
||||||
|
}
|
||||||
|
if ($result['unreadable'] !== []) {
|
||||||
|
$output->writeln('<error>' . count($result['unreadable']) . ' blob(s) could not be read from the location</error> (often a temporary storage '
|
||||||
|
. 'or network error: run the check again; if the same blobs stay unreadable, their pack may be damaged or truncated): '
|
||||||
|
. implode(', ', $result['unreadable']));
|
||||||
|
$output->writeln('Last read error: ' . $result['readError']);
|
||||||
|
}
|
||||||
|
if ($result['missing'] === [] && $result['failed'] === [] && $result['unreadable'] === []) {
|
||||||
|
$output->writeln('<info>OK</info>');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,310 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Controller;
|
||||||
|
|
||||||
|
use OCA\NgBackup\AppInfo\Application;
|
||||||
|
use OCA\NgBackup\BackgroundJob\RestoreJob;
|
||||||
|
use OCA\NgBackup\Command\KeyConfirm;
|
||||||
|
use OCA\NgBackup\Db\Run;
|
||||||
|
use OCA\NgBackup\Db\RunMapper;
|
||||||
|
use OCA\NgBackup\Db\SnapshotMapper;
|
||||||
|
use OCA\NgBackup\Service\BackupService;
|
||||||
|
use OCA\NgBackup\Service\KeyService;
|
||||||
|
use OCA\NgBackup\Service\RestoreService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCP\AppFramework\Controller;
|
||||||
|
use OCP\AppFramework\Http;
|
||||||
|
use OCP\AppFramework\Http\Attribute\NoCSRFRequired;
|
||||||
|
use OCP\AppFramework\Http\Attribute\PasswordConfirmationRequired;
|
||||||
|
use OCP\AppFramework\Http\DataDownloadResponse;
|
||||||
|
use OCP\AppFramework\Http\JSONResponse;
|
||||||
|
use OCP\BackgroundJob\IJobList;
|
||||||
|
use OCP\IAppConfig;
|
||||||
|
use OCP\IRequest;
|
||||||
|
use OCP\IUserSession;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSON API for the admin settings page. Every method requires an admin (the AppFramework
|
||||||
|
* default for controllers without NoAdminRequired) and a valid CSRF token.
|
||||||
|
*/
|
||||||
|
final class AdminApiController extends Controller {
|
||||||
|
public function __construct(
|
||||||
|
IRequest $request,
|
||||||
|
private KeyService $keys,
|
||||||
|
private TargetService $targets,
|
||||||
|
private BackupService $backups,
|
||||||
|
private RestoreService $restore,
|
||||||
|
private RunMapper $runs,
|
||||||
|
private SnapshotMapper $snapshots,
|
||||||
|
private IAppConfig $appConfig,
|
||||||
|
private IJobList $jobs,
|
||||||
|
private IUserSession $session,
|
||||||
|
private \OCA\NgBackup\Service\AlertService $alerts,
|
||||||
|
private \OCA\NgBackup\Service\KeySlotService $slots,
|
||||||
|
private \OCA\NgBackup\Service\PruneService $prune,
|
||||||
|
) {
|
||||||
|
parent::__construct(Application::APP_ID, $request);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function status(): JSONResponse {
|
||||||
|
$targets = [];
|
||||||
|
foreach ($this->targets->list() as $t) {
|
||||||
|
$targets[] = ['id' => $t->getId(), 'name' => $t->getName(), 'backend' => $t->getBackend(), 'path' => $t->getBasePath(),
|
||||||
|
'repositoryId' => $t->getRepositoryId(), 'createdAt' => $t->getCreatedAt(), 'appendOnly' => (bool)$t->getAppendOnly(),
|
||||||
|
'maxBytes' => $t->getMaxBytes(), 'storedBytes' => $this->alerts->lastStoredBytes($t)];
|
||||||
|
}
|
||||||
|
$runs = array_map(fn (Run $r) => $this->runJson($r), $this->runs->findRecent(15));
|
||||||
|
return new JSONResponse([
|
||||||
|
'key' => ['initialized' => $this->keys->isInitialized(), 'fingerprint' => $this->keys->fingerprint(),
|
||||||
|
'confirmation' => $this->keys->recoveryKitConfirmation(), 'everConfirmed' => $this->keys->isInitialized() && $this->keys->everConfirmed(),
|
||||||
|
'slots' => $this->keys->isInitialized() ? $this->keys->slots() : [], 'maxSlots' => \OCA\NgBackup\Service\KeyService::MAX_SLOTS,
|
||||||
|
'deleteDelayDays' => $this->keys->isInitialized() ? $this->keys->deleteDelayDays() : \OCA\NgBackup\Service\KeyService::DEFAULT_DELETE_DELAY],
|
||||||
|
'statement' => KeyConfirm::STATEMENT,
|
||||||
|
'confirmationPhrase' => KeyConfirm::CONFIRMATION,
|
||||||
|
'targets' => $targets,
|
||||||
|
'runs' => $runs,
|
||||||
|
'schedule' => $this->appConfig->getValueString(Application::APP_ID, 'schedule_time', ''),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function initKey(string $passphrase, int $deleteDelayDays = \OCA\NgBackup\Service\KeyService::DEFAULT_DELETE_DELAY, string $label = 'Slot 1'): JSONResponse {
|
||||||
|
try {
|
||||||
|
$this->keys->initialize($passphrase, $deleteDelayDays, $label);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
return new JSONResponse(['fingerprint' => $this->keys->fingerprint()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Plain download link (no CSRF token on a navigation), therefore guarded by a recent password
|
||||||
|
* confirmation instead, logged to the audit log and announced to all administrators. The kit
|
||||||
|
* only holds the passphrase-wrapped key: useless without the passphrase.
|
||||||
|
*/
|
||||||
|
#[NoCSRFRequired]
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function downloadKit(): DataDownloadResponse|JSONResponse {
|
||||||
|
if (!$this->keys->isInitialized()) {
|
||||||
|
return new JSONResponse(['error' => 'No backup key yet'], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
$this->alerts->securityEvent('kit_downloaded');
|
||||||
|
$json = json_encode($this->keys->recoveryKit(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
||||||
|
return new DataDownloadResponse($json, 'ng_backup-recovery-kit-' . $this->keys->fingerprint() . '.json', 'application/json');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The admin must tick the statement, send the exact phrase and the code from the kit file. */
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function confirmKit(bool $accepted, string $phrase, string $code = ''): JSONResponse {
|
||||||
|
if (!$accepted || $phrase !== KeyConfirm::CONFIRMATION) {
|
||||||
|
return new JSONResponse(['error' => 'Not confirmed'], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
if (!$this->keys->checkKitCode($code)) {
|
||||||
|
return new JSONResponse(['error' => 'The confirmation code does not match the current recovery kit'], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
$this->keys->confirmRecoveryKit($this->session->getUser()?->getUID() ?? 'unknown');
|
||||||
|
return new JSONResponse(['confirmation' => $this->keys->recoveryKitConfirmation()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function addSlot(string $passphrase, string $label): JSONResponse {
|
||||||
|
try {
|
||||||
|
return new JSONResponse(['slot' => $this->slots->add($passphrase, $label)]);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function replaceSlot(int $slot, string $passphrase, ?string $label = null): JSONResponse {
|
||||||
|
try {
|
||||||
|
$this->slots->replace($slot, $passphrase, $label);
|
||||||
|
return new JSONResponse([]);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function removeSlot(int $slot): JSONResponse {
|
||||||
|
try {
|
||||||
|
$this->slots->remove($slot);
|
||||||
|
return new JSONResponse([]);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function trash(int $targetId): JSONResponse {
|
||||||
|
try {
|
||||||
|
$delay = $this->keys->deleteDelayDays() * 86400;
|
||||||
|
$out = [];
|
||||||
|
foreach ($this->targets->repository($this->targets->get((string)$targetId))->trash() as $id => $info) {
|
||||||
|
$out[] = ['id' => $id, 'label' => $info['label'], 'time' => $info['time'], 'removedAt' => $info['forgottenAt'],
|
||||||
|
'by' => $info['by'], 'deleteAfter' => $info['forgottenAt'] + $delay];
|
||||||
|
}
|
||||||
|
return new JSONResponse($out);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function untrash(int $targetId, string $snapshotId): JSONResponse {
|
||||||
|
try {
|
||||||
|
$this->prune->untrash($this->targets->get((string)$targetId), $snapshotId);
|
||||||
|
return new JSONResponse([]);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function backends(): JSONResponse {
|
||||||
|
return new JSONResponse($this->targets->availableBackends());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function addTarget(string $name, string $backend, string $auth, array $options = [], string $path = 'ng_backup', ?float $maxGb = null): JSONResponse {
|
||||||
|
try {
|
||||||
|
$r = $this->targets->add($name, $backend, $auth, $options, $path);
|
||||||
|
if ($maxGb !== null && $maxGb > 0) {
|
||||||
|
$this->targets->setMaxBytes($r['target'], (int)round($maxGb * 1073741824));
|
||||||
|
}
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
return new JSONResponse(['id' => $r['target']->getId(), 'created' => $r['created']]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The stored settings for the edit form; password values are left out (null). */
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function targetSettings(int $id): JSONResponse {
|
||||||
|
$t = $this->targets->get((string)$id);
|
||||||
|
return new JSONResponse(['name' => $t->getName(), 'backend' => $t->getBackend(), 'auth' => $t->getAuth(),
|
||||||
|
'options' => $this->targets->editableOptions($t), 'path' => $t->getBasePath()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Options with a null value keep their stored value. */
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function updateTarget(int $id, string $name, string $backend, string $auth, array $options = [], string $path = 'ng_backup'): JSONResponse {
|
||||||
|
try {
|
||||||
|
$t = $this->targets->get((string)$id);
|
||||||
|
$before = $t->getName();
|
||||||
|
$t = $this->targets->update($t, $name, $backend, $auth, $options, $path);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
$this->alerts->securityEvent('target_changed', ['target' => $before]);
|
||||||
|
return new JSONResponse(['id' => $t->getId()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Size limit in GiB; null or 0 = unlimited. */
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function setTargetLimit(int $id, ?float $maxGb = null): JSONResponse {
|
||||||
|
$t = $this->targets->setMaxBytes($this->targets->get((string)$id), ($maxGb !== null && $maxGb > 0) ? (int)round($maxGb * 1073741824) : null);
|
||||||
|
return new JSONResponse(['maxBytes' => $t->getMaxBytes()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function removeTarget(int $id): JSONResponse {
|
||||||
|
$target = $this->targets->get((string)$id);
|
||||||
|
$this->targets->remove($target);
|
||||||
|
$this->alerts->securityEvent('target_removed', ['target' => $target->getName()]);
|
||||||
|
return new JSONResponse([]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function testTarget(int $id): JSONResponse {
|
||||||
|
try {
|
||||||
|
$repo = $this->targets->repository($this->targets->get((string)$id));
|
||||||
|
return new JSONResponse(['ok' => true, 'snapshots' => count($repo->listSnapshots())]);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['ok' => false, 'error' => $e->getMessage()]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function startBackup(int $targetId, string $label = ''): JSONResponse {
|
||||||
|
try {
|
||||||
|
$run = $this->backups->start($this->targets->get((string)$targetId), $label);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
return new JSONResponse($this->runJson($run));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setSchedule(string $time): JSONResponse {
|
||||||
|
if ($time !== '' && !preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $time)) {
|
||||||
|
return new JSONResponse(['error' => 'Use HH:MM or empty'], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, 'schedule_time', $time);
|
||||||
|
return new JSONResponse(['schedule' => $time]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function snapshots(int $targetId): JSONResponse {
|
||||||
|
$out = [];
|
||||||
|
foreach ($this->snapshots->findForTarget($targetId) as $s) {
|
||||||
|
$out[] = ['id' => $s->getSnapshotId(), 'kind' => $s->getKind(), 'label' => $s->getLabel(), 'createdAt' => $s->getCreatedAt(),
|
||||||
|
'files' => $s->getFiles(), 'bytes' => $s->getBytes()];
|
||||||
|
}
|
||||||
|
return new JSONResponse($out);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function browse(int $targetId, string $snapshotId, string $path = ''): JSONResponse {
|
||||||
|
try {
|
||||||
|
return new JSONResponse($this->restore->browse($this->targets->get((string)$targetId), $snapshotId, $path));
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Queue a restore of user files; it runs as a background job and shows up in the runs. */
|
||||||
|
#[PasswordConfirmationRequired]
|
||||||
|
public function startRestore(int $targetId, string $snapshotId, string $path, string $mode = RestoreService::MODE_NEW_FOLDER): JSONResponse {
|
||||||
|
if (!in_array($mode, [RestoreService::MODE_NEW_FOLDER, RestoreService::MODE_MERGE, RestoreService::MODE_REPLACE], true)
|
||||||
|
|| !\OCA\NgBackup\Repository\Repository::isSafePath($path) || !preg_match('#^data/[^/]+/files(/.*)?$#', $path)) {
|
||||||
|
return new JSONResponse(['error' => 'Invalid path or mode'], Http::STATUS_BAD_REQUEST);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$target = $this->targets->get((string)$targetId);
|
||||||
|
} catch (\Throwable) {
|
||||||
|
return new JSONResponse(['error' => 'No such location'], Http::STATUS_NOT_FOUND);
|
||||||
|
}
|
||||||
|
$run = new Run();
|
||||||
|
$run->setTargetId($target->getId());
|
||||||
|
$run->setKind('restore');
|
||||||
|
$run->setStatus(Run::RUNNING);
|
||||||
|
$run->setPhase('queued');
|
||||||
|
$run->setState(json_encode(['snapshot' => $snapshotId, 'path' => $path, 'mode' => $mode,
|
||||||
|
'by' => $this->session->getUser()?->getUID()], JSON_THROW_ON_ERROR));
|
||||||
|
$run->setSnapshotId($snapshotId);
|
||||||
|
$run->setStartedAt(time());
|
||||||
|
$run->setUpdatedAt(time());
|
||||||
|
try {
|
||||||
|
$run = $this->prune->underStartLock($target, fn () => $this->runs->insert($run));
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_CONFLICT);
|
||||||
|
}
|
||||||
|
$this->jobs->add(RestoreJob::class, ['run' => $run->getId()]);
|
||||||
|
return new JSONResponse($this->runJson($run));
|
||||||
|
}
|
||||||
|
|
||||||
|
private function runJson(Run $r): array {
|
||||||
|
$snapshot = null;
|
||||||
|
if ($r->getSnapshotId() !== null) {
|
||||||
|
try {
|
||||||
|
$s = $this->snapshots->findOne($r->getTargetId(), $r->getSnapshotId());
|
||||||
|
$snapshot = ['createdAt' => $s->getCreatedAt(), 'label' => $s->getLabel()];
|
||||||
|
} catch (\Throwable) {
|
||||||
|
// Snapshot already pruned or trashed: the run keeps its id, just not the date.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ['id' => $r->getId(), 'targetId' => $r->getTargetId(), 'kind' => $r->getKind(), 'status' => $r->getStatus(),
|
||||||
|
'phase' => $r->getPhase(), 'snapshotId' => $r->getSnapshotId(), 'snapshot' => $snapshot, 'startedAt' => $r->getStartedAt(),
|
||||||
|
'updatedAt' => $r->getUpdatedAt(), 'finishedAt' => $r->getFinishedAt(), 'error' => $r->getError(),
|
||||||
|
'stats' => json_decode($r->getStats() ?? 'null', true), 'progress' => \OCA\NgBackup\Job\BackupRun::progressOf($r)];
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,5 +7,5 @@ declare(strict_types=1);
|
|||||||
|
|
||||||
namespace OCA\NgBackup\Crypto;
|
namespace OCA\NgBackup\Crypto;
|
||||||
|
|
||||||
class CryptoException extends \RuntimeException {
|
final class CryptoException extends \RuntimeException {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,9 +76,32 @@ final class KeyRing {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open a key field that is either one wrapped key or {"slots": [{slot, label, key}, ...]}:
|
||||||
|
* every slot holds the same master key under its own passphrase.
|
||||||
|
*/
|
||||||
|
public static function unwrapAny(array $field, #[\SensitiveParameter] string $passphrase): self {
|
||||||
|
if (!isset($field['slots'])) {
|
||||||
|
return self::unwrap($field, $passphrase);
|
||||||
|
}
|
||||||
|
foreach ($field['slots'] as $slot) {
|
||||||
|
try {
|
||||||
|
return self::unwrap($slot['key'], $passphrase);
|
||||||
|
} catch (CryptoException) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CryptoException('Wrong passphrase or damaged key');
|
||||||
|
}
|
||||||
|
|
||||||
/** @param array{v:int, salt:string, ops:int, mem:int, nonce:string, key:string} $wrapped */
|
/** @param array{v:int, salt:string, ops:int, mem:int, nonce:string, key:string} $wrapped */
|
||||||
public static function unwrap(array $wrapped, #[\SensitiveParameter] string $passphrase): self {
|
public static function unwrap(array $wrapped, #[\SensitiveParameter] string $passphrase): self {
|
||||||
$b64 = static fn (string $s): string => sodium_base642bin($s, SODIUM_BASE64_VARIANT_ORIGINAL);
|
$b64 = static fn (string $s): string => sodium_base642bin($s, SODIUM_BASE64_VARIANT_ORIGINAL);
|
||||||
|
// ops/mem come from unauthenticated JSON (kit file, repository config): bound them, so a
|
||||||
|
// damaged or crafted file cannot make the KDF use unbounded memory or time.
|
||||||
|
if ($wrapped['ops'] < SODIUM_CRYPTO_PWHASH_OPSLIMIT_INTERACTIVE || $wrapped['ops'] > SODIUM_CRYPTO_PWHASH_OPSLIMIT_SENSITIVE
|
||||||
|
|| $wrapped['mem'] < SODIUM_CRYPTO_PWHASH_MEMLIMIT_INTERACTIVE || $wrapped['mem'] > SODIUM_CRYPTO_PWHASH_MEMLIMIT_SENSITIVE) {
|
||||||
|
throw new CryptoException('Unsupported key derivation parameters (damaged or foreign key file)');
|
||||||
|
}
|
||||||
$kek = sodium_crypto_pwhash(SODIUM_CRYPTO_SECRETBOX_KEYBYTES, $passphrase, $b64($wrapped['salt']),
|
$kek = sodium_crypto_pwhash(SODIUM_CRYPTO_SECRETBOX_KEYBYTES, $passphrase, $b64($wrapped['salt']),
|
||||||
$wrapped['ops'], $wrapped['mem'], SODIUM_CRYPTO_PWHASH_ALG_ARGON2ID13);
|
$wrapped['ops'], $wrapped['mem'], SODIUM_CRYPTO_PWHASH_ALG_ARGON2ID13);
|
||||||
$master = sodium_crypto_secretbox_open($b64($wrapped['key']), $b64($wrapped['nonce']), $kek);
|
$master = sodium_crypto_secretbox_open($b64($wrapped['key']), $b64($wrapped['nonce']), $kek);
|
||||||
|
|||||||
@@ -122,6 +122,7 @@ final class StreamCipher {
|
|||||||
/** fread() may return short reads on network streams; keep reading until $length or EOF. */
|
/** fread() may return short reads on network streams; keep reading until $length or EOF. */
|
||||||
private static function readFull($in, int $length): string {
|
private static function readFull($in, int $length): string {
|
||||||
$buffer = '';
|
$buffer = '';
|
||||||
|
$empty = 0;
|
||||||
while (strlen($buffer) < $length && !feof($in)) {
|
while (strlen($buffer) < $length && !feof($in)) {
|
||||||
$chunk = fread($in, $length - strlen($buffer));
|
$chunk = fread($in, $length - strlen($buffer));
|
||||||
if ($chunk === false) {
|
if ($chunk === false) {
|
||||||
@@ -132,8 +133,15 @@ final class StreamCipher {
|
|||||||
if (feof($in)) {
|
if (feof($in)) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// A stalled network stream returns '' without EOF forever: back off, then give up
|
||||||
|
// (about a minute) instead of spinning at full CPU until the lease expires.
|
||||||
|
if (++$empty > 600) {
|
||||||
|
throw new CryptoException('Read stalled');
|
||||||
|
}
|
||||||
|
usleep(100000);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
$empty = 0;
|
||||||
$buffer .= $chunk;
|
$buffer .= $chunk;
|
||||||
}
|
}
|
||||||
return $buffer;
|
return $buffer;
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Db;
|
||||||
|
|
||||||
|
use OCA\NgBackup\AppInfo\Application;
|
||||||
|
use OCA\NgBackup\Repository\ICatalogAnchor;
|
||||||
|
use OCP\IAppConfig;
|
||||||
|
|
||||||
|
/** Keeps the newest catalog generation seen per repository in the app config (outside the location). */
|
||||||
|
final class AppConfigCatalogAnchor implements ICatalogAnchor {
|
||||||
|
public function __construct(
|
||||||
|
private IAppConfig $appConfig,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public function get(string $repositoryId): ?array {
|
||||||
|
$raw = $this->appConfig->getValueString(Application::APP_ID, $this->key($repositoryId), '', true);
|
||||||
|
$data = $raw === '' ? null : json_decode($raw, true);
|
||||||
|
return is_array($data) && isset($data['gen'], $data['hash']) ? ['gen' => (int)$data['gen'], 'hash' => (string)$data['hash']] : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function set(string $repositoryId, int $gen, string $hash): void {
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, $this->key($repositoryId), json_encode(['gen' => $gen, 'hash' => $hash]), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function key(string $repositoryId): string {
|
||||||
|
return 'catalog_' . preg_replace('/[^a-f0-9]/', '', $repositoryId);
|
||||||
|
}
|
||||||
|
}
|
||||||
+133
-43
@@ -31,6 +31,8 @@ use OCA\NgBackup\Repository\Repository;
|
|||||||
*/
|
*/
|
||||||
final class DbDumper {
|
final class DbDumper {
|
||||||
private const BATCH = 2000;
|
private const BATCH = 2000;
|
||||||
|
/** Rows per INSERT on restore; keeps the number of bound parameters well below database limits. */
|
||||||
|
private const INSERT_ROWS = 100;
|
||||||
/** Tables without a primary key are read in one query; refuse that above this size. */
|
/** Tables without a primary key are read in one query; refuse that above this size. */
|
||||||
private const MAX_ROWS_WITHOUT_PK = 50000;
|
private const MAX_ROWS_WITHOUT_PK = 50000;
|
||||||
|
|
||||||
@@ -41,39 +43,94 @@ final class DbDumper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return array{tables: array<string, array{blobs: list<string>, rows: int, bytes: int, sha256: string}>, provider: string}
|
* Dump everything in one go (one transaction, fully consistent).
|
||||||
|
*
|
||||||
|
* @return array{tables: array<string, array{blobs: list<string>, rows: int, bytes: int, sha256: string}>, sequences: array, provider: string, consistent: bool}
|
||||||
*/
|
*/
|
||||||
public function dump(Repository $repo, array &$stats): array {
|
public function dump(Repository $repo, array &$stats): array {
|
||||||
$sm = $this->db->createSchemaManager();
|
$state = $this->startDump();
|
||||||
$tables = array_values(array_filter($sm->listTableNames(), fn (string $t) => str_starts_with($t, $this->prefix)));
|
while (!$state['done']) {
|
||||||
sort($tables);
|
$state = $this->dumpStep($repo, $state, PHP_FLOAT_MAX, $stats);
|
||||||
|
}
|
||||||
|
return $this->manifest($state);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** State for a resumable dump (see dumpStep()). */
|
||||||
|
public function startDump(): array {
|
||||||
|
$tables = array_values(array_filter($this->db->createSchemaManager()->listTableNames(), fn (string $t) => str_starts_with($t, $this->prefix)));
|
||||||
|
sort($tables);
|
||||||
|
return ['tables' => $tables, 'i' => 0, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null,
|
||||||
|
'result' => [], 'sequences' => [], 'steps' => 0, 'done' => false];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dump tables until $deadline, inside one REPEATABLE READ transaction for this step. A table
|
||||||
|
* that does not fit is resumed in the next step at the last primary key written. If the whole
|
||||||
|
* dump fits in one step it is fully consistent; otherwise each step is consistent on its own
|
||||||
|
* ('consistent' => false in the manifest).
|
||||||
|
*/
|
||||||
|
public function dumpStep(Repository $repo, array $state, float $deadline, array &$stats): array {
|
||||||
|
$state['steps']++;
|
||||||
|
$sm = $this->db->createSchemaManager();
|
||||||
|
// A pack that fills up during the dump is uploaded right away; its index cache row must
|
||||||
|
// wait until the read-only transaction is over.
|
||||||
|
$repo->holdIndexCache();
|
||||||
|
try {
|
||||||
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
|
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
|
||||||
$this->db->beginTransaction();
|
$this->db->beginTransaction();
|
||||||
try {
|
try {
|
||||||
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
|
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
|
||||||
$this->db->executeStatement('SET TRANSACTION READ ONLY');
|
$this->db->executeStatement('SET TRANSACTION READ ONLY');
|
||||||
}
|
}
|
||||||
$result = [];
|
while ($state['i'] < count($state['tables'])) {
|
||||||
foreach ($tables as $table) {
|
$table = $state['tables'][$state['i']];
|
||||||
|
$schema = $sm->introspectTable($table);
|
||||||
$writer = $repo->blobWriter($stats);
|
$writer = $repo->blobWriter($stats);
|
||||||
$hash = hash_init('sha256');
|
$hash = $state['hash'] !== null ? unserialize(base64_decode($state['hash']), ['allowed_classes' => [\HashContext::class]]) : hash_init('sha256');
|
||||||
$rows = 0;
|
if ($state['after'] === null && $state['segments'] === []) {
|
||||||
foreach ($this->tableLines($sm->introspectTable($table)) as $kind => $line) {
|
$writer->write($this->header($schema) . "\n");
|
||||||
|
}
|
||||||
|
$complete = true;
|
||||||
|
foreach ($this->rowLines($schema, $state['after']) as [$key, $line]) {
|
||||||
$writer->write($line . "\n");
|
$writer->write($line . "\n");
|
||||||
if ($kind === 'row') {
|
|
||||||
hash_update($hash, $line . "\n");
|
hash_update($hash, $line . "\n");
|
||||||
$rows++;
|
$state['rows']++;
|
||||||
|
$state['after'] = $key;
|
||||||
|
if (microtime(true) >= $deadline) {
|
||||||
|
$complete = false;
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$result[$table] = ['blobs' => $writer->finish(), 'rows' => $rows, 'bytes' => $writer->bytes(), 'sha256' => hash_final($hash)];
|
$state['segments'][] = $writer->finish();
|
||||||
|
$state['bytes'] += $writer->bytes();
|
||||||
|
if (!$complete) {
|
||||||
|
$state['hash'] = base64_encode(serialize($hash));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
$state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'],
|
||||||
|
'bytes' => $state['bytes'], 'sha256' => hash_final($hash)];
|
||||||
|
$state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]);
|
||||||
|
if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($state['i'] >= count($state['tables'])) {
|
||||||
|
$state['sequences'] = $this->sequenceStates($state['tables']);
|
||||||
|
$state['done'] = true;
|
||||||
}
|
}
|
||||||
$sequences = $this->sequenceStates($tables);
|
|
||||||
} finally {
|
} finally {
|
||||||
$this->db->rollBack();
|
$this->db->rollBack();
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
$repo->releaseIndexCache();
|
||||||
|
}
|
||||||
$repo->flushPacks();
|
$repo->flushPacks();
|
||||||
return ['tables' => $result, 'sequences' => $sequences, 'provider' => get_class($this->db->getDatabasePlatform())];
|
return $state;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function manifest(array $state): array {
|
||||||
|
return ['tables' => $state['result'], 'sequences' => $state['sequences'],
|
||||||
|
'provider' => get_class($this->db->getDatabasePlatform()), 'consistent' => $state['steps'] === 1];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -82,16 +139,23 @@ final class DbDumper {
|
|||||||
*
|
*
|
||||||
* @param array{tables: array<string, array{blobs: list<string>, rows: int}>} $manifest
|
* @param array{tables: array<string, array{blobs: list<string>, rows: int}>} $manifest
|
||||||
*/
|
*/
|
||||||
public function restore(Repository $repo, array $manifest): int {
|
public function restore(Repository $repo, array $manifest, ?array &$report = null, ?callable $heartbeat = null): int {
|
||||||
$sm = $this->db->createSchemaManager();
|
$sm = $this->db->createSchemaManager();
|
||||||
$existing = array_flip($sm->listTableNames());
|
$existing = array_flip($sm->listTableNames());
|
||||||
$total = 0;
|
$total = 0;
|
||||||
|
$report = ['skipped' => [], 'restored' => []];
|
||||||
$this->db->beginTransaction();
|
$this->db->beginTransaction();
|
||||||
try {
|
try {
|
||||||
foreach ($manifest['tables'] as $table => $info) {
|
foreach ($manifest['tables'] as $table => $info) {
|
||||||
if (!isset($existing[$table])) {
|
if (!isset($existing[$table])) {
|
||||||
throw new \RuntimeException("Table $table from the backup does not exist here (different app versions?)");
|
throw new \RuntimeException("Table $table from the backup does not exist here (different app versions?)");
|
||||||
}
|
}
|
||||||
|
// Unchanged since the backup: leave it alone (much faster than delete + insert).
|
||||||
|
$live = $this->checksum($table);
|
||||||
|
if ($live['rows'] === $info['rows'] && hash_equals($info['sha256'], $live['sha256'])) {
|
||||||
|
$report['skipped'][] = $table;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
$this->db->executeStatement('DELETE FROM ' . $this->db->quoteIdentifier($table));
|
$this->db->executeStatement('DELETE FROM ' . $this->db->quoteIdentifier($table));
|
||||||
$columns = null;
|
$columns = null;
|
||||||
$binary = [];
|
$binary = [];
|
||||||
@@ -112,17 +176,21 @@ final class DbDumper {
|
|||||||
if (is_array($value) && isset($value['$b'])) {
|
if (is_array($value) && isset($value['$b'])) {
|
||||||
$value = base64_decode($value['$b'], true); // text that was not valid UTF-8
|
$value = base64_decode($value['$b'], true); // text that was not valid UTF-8
|
||||||
}
|
}
|
||||||
$row[$col] = ($value !== null && isset($binary[$col])) ? base64_decode($value, true) : $value;
|
$row[] = ($value !== null && isset($binary[$col])) ? base64_decode($value, true) : $value;
|
||||||
}
|
}
|
||||||
$batch[] = $row;
|
$batch[] = $row;
|
||||||
if (count($batch) >= 500) {
|
if (count($batch) >= self::INSERT_ROWS) {
|
||||||
$total += $this->insertBatch($table, $batch, $binary);
|
$total += $this->insertBatch($table, $columns, $batch, $binary);
|
||||||
$batch = [];
|
$batch = [];
|
||||||
|
if ($heartbeat !== null) {
|
||||||
|
$heartbeat();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ($batch !== []) {
|
if ($batch !== []) {
|
||||||
$total += $this->insertBatch($table, $batch, $binary);
|
$total += $this->insertBatch($table, $columns, $batch, $binary);
|
||||||
}
|
}
|
||||||
|
$report['restored'][] = $table;
|
||||||
}
|
}
|
||||||
$this->restoreSequences($manifest['sequences'] ?? []);
|
$this->restoreSequences($manifest['sequences'] ?? []);
|
||||||
$this->db->commit();
|
$this->db->commit();
|
||||||
@@ -135,20 +203,22 @@ final class DbDumper {
|
|||||||
|
|
||||||
/** Same checksum as dump(): sha256 over the row lines in primary-key order. */
|
/** Same checksum as dump(): sha256 over the row lines in primary-key order. */
|
||||||
public function checksum(string $table): array {
|
public function checksum(string $table): array {
|
||||||
$sm = $this->db->createSchemaManager();
|
|
||||||
$hash = hash_init('sha256');
|
$hash = hash_init('sha256');
|
||||||
$rows = 0;
|
$rows = 0;
|
||||||
foreach ($this->tableLines($sm->introspectTable($table)) as $kind => $line) {
|
foreach ($this->rowLines($this->db->createSchemaManager()->introspectTable($table)) as [, $line]) {
|
||||||
if ($kind === 'row') {
|
|
||||||
hash_update($hash, $line . "\n");
|
hash_update($hash, $line . "\n");
|
||||||
$rows++;
|
$rows++;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return ['rows' => $rows, 'sha256' => hash_final($hash)];
|
return ['rows' => $rows, 'sha256' => hash_final($hash)];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return \Generator<string, string> 'header' => line, then 'row' => line ... */
|
private function header(\Doctrine\DBAL\Schema\Table $table): string {
|
||||||
private function tableLines(\Doctrine\DBAL\Schema\Table $table): \Generator {
|
[$columns, $binary, $pk] = $this->describe($table);
|
||||||
|
return json_encode(['table' => $table->getName(), 'columns' => $columns, 'binary' => $binary, 'pk' => $pk], JSON_THROW_ON_ERROR);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return array{0: list<string>, 1: list<string>, 2: list<string>} columns, binary columns, primary key */
|
||||||
|
private function describe(\Doctrine\DBAL\Schema\Table $table): array {
|
||||||
$columns = array_values(array_map(fn ($c) => $c->getName(), $table->getColumns()));
|
$columns = array_values(array_map(fn ($c) => $c->getName(), $table->getColumns()));
|
||||||
$binary = [];
|
$binary = [];
|
||||||
foreach ($table->getColumns() as $c) {
|
foreach ($table->getColumns() as $c) {
|
||||||
@@ -156,25 +226,34 @@ final class DbDumper {
|
|||||||
$binary[] = $c->getName();
|
$binary[] = $c->getName();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$pk = $table->getPrimaryKey()?->getColumns() ?? [];
|
return [$columns, $binary, $table->getPrimaryKey()?->getColumns() ?? []];
|
||||||
yield 'header' => json_encode(['table' => $table->getName(), 'columns' => array_values($columns), 'binary' => $binary, 'pk' => $pk], JSON_THROW_ON_ERROR);
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rows in primary-key order, after the key $after (resume), as [key, JSON line].
|
||||||
|
*
|
||||||
|
* @return \Generator<array{0: list<mixed>|null, 1: string}>
|
||||||
|
*/
|
||||||
|
private function rowLines(\Doctrine\DBAL\Schema\Table $table, ?array $after = null): \Generator {
|
||||||
|
[$columns, $binary, $pk] = $this->describe($table);
|
||||||
$binaryFlip = array_flip($binary);
|
$binaryFlip = array_flip($binary);
|
||||||
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
||||||
$select = 'SELECT ' . implode(', ', array_map($q, $columns)) . ' FROM ' . $q($table->getName());
|
$select = 'SELECT ' . implode(', ', array_map($q, $columns)) . ' FROM ' . $q($table->getName());
|
||||||
|
|
||||||
if ($pk === []) {
|
if ($pk === []) {
|
||||||
// Nextcloud requires primary keys; small odd tables are read in one query.
|
// Nextcloud requires primary keys; small odd tables are read in one query (not resumable).
|
||||||
$count = (int)$this->db->fetchOne('SELECT COUNT(*) FROM ' . $q($table->getName()));
|
$count = (int)$this->db->fetchOne('SELECT COUNT(*) FROM ' . $q($table->getName()));
|
||||||
if ($count > self::MAX_ROWS_WITHOUT_PK) {
|
if ($count > self::MAX_ROWS_WITHOUT_PK) {
|
||||||
throw new \RuntimeException("Table {$table->getName()} has no primary key and $count rows; cannot dump it with bounded memory");
|
throw new \RuntimeException("Table {$table->getName()} has no primary key and $count rows; cannot dump it with bounded memory");
|
||||||
}
|
}
|
||||||
$batches = [$this->db->fetchAllNumeric($select)];
|
$batches = [$this->db->fetchAllNumeric($select)];
|
||||||
|
$pkIdx = [];
|
||||||
} else {
|
} else {
|
||||||
$batches = $this->keysetBatches($select, $pk, $columns);
|
$batches = $this->keysetBatches($select, $pk, $columns, $after);
|
||||||
|
$pkIdx = array_map(fn ($c) => array_search($c, $columns, true), $pk);
|
||||||
}
|
}
|
||||||
foreach ($batches as $rows) {
|
foreach ($batches as $rows) {
|
||||||
foreach ($rows as $row) {
|
foreach ($rows as $row) {
|
||||||
|
$key = $pkIdx === [] ? null : array_map(fn ($i) => $row[$i], $pkIdx);
|
||||||
foreach ($columns as $i => $col) {
|
foreach ($columns as $i => $col) {
|
||||||
if (is_resource($row[$i])) {
|
if (is_resource($row[$i])) {
|
||||||
$row[$i] = stream_get_contents($row[$i]);
|
$row[$i] = stream_get_contents($row[$i]);
|
||||||
@@ -184,28 +263,32 @@ final class DbDumper {
|
|||||||
} elseif (is_string($row[$i]) && !mb_check_encoding($row[$i], 'UTF-8')) {
|
} elseif (is_string($row[$i]) && !mb_check_encoding($row[$i], 'UTF-8')) {
|
||||||
// Keep invalid UTF-8 byte-exact instead of substituting characters.
|
// Keep invalid UTF-8 byte-exact instead of substituting characters.
|
||||||
$row[$i] = ['$b' => base64_encode($row[$i])];
|
$row[$i] = ['$b' => base64_encode($row[$i])];
|
||||||
} elseif (is_int($row[$i]) || is_float($row[$i]) || is_bool($row[$i])) {
|
} elseif (is_float($row[$i])) {
|
||||||
|
// var_export uses serialize_precision (shortest exact form), not the lossy
|
||||||
|
// 14-digit 'precision' of a (string) cast.
|
||||||
|
$row[$i] = var_export($row[$i], true);
|
||||||
|
} elseif (is_int($row[$i]) || is_bool($row[$i])) {
|
||||||
// Normalise: drivers differ in returning numbers as int or string.
|
// Normalise: drivers differ in returning numbers as int or string.
|
||||||
$row[$i] = (string)(is_bool($row[$i]) ? (int)$row[$i] : $row[$i]);
|
$row[$i] = (string)(is_bool($row[$i]) ? (int)$row[$i] : $row[$i]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
yield 'row' => json_encode($row, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR);
|
yield [$key, json_encode($row, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR)];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return \Generator<list<list<mixed>>> */
|
/** @return \Generator<list<list<mixed>>> */
|
||||||
private function keysetBatches(string $select, array $pk, array $columns): \Generator {
|
private function keysetBatches(string $select, array $pk, array $columns, ?array $after): \Generator {
|
||||||
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
||||||
$order = ' ORDER BY ' . implode(', ', array_map($q, $pk));
|
$order = ' ORDER BY ' . implode(', ', array_map($q, $pk));
|
||||||
$pkIdx = array_map(fn ($c) => array_search($c, $columns, true), $pk);
|
$cols = array_map(fn ($c) => $q($c), $pk);
|
||||||
$last = null;
|
$last = $after;
|
||||||
while (true) {
|
while (true) {
|
||||||
$sql = $select;
|
$sql = $select;
|
||||||
$params = [];
|
$params = [];
|
||||||
if ($last !== null) {
|
if ($last !== null) {
|
||||||
// Row-value comparison (a, b) > (?, ?), supported by PostgreSQL, MySQL/MariaDB and SQLite.
|
// Row-value comparison (a, b) > (?, ?), supported by PostgreSQL, MySQL/MariaDB and SQLite.
|
||||||
$sql .= ' WHERE (' . implode(', ', array_map($q, $pk)) . ') > (' . implode(', ', array_fill(0, count($pk), '?')) . ')';
|
$sql .= ' WHERE (' . implode(', ', $cols) . ') > (' . implode(', ', array_fill(0, count($pk), '?')) . ')';
|
||||||
$params = $last;
|
$params = $last;
|
||||||
}
|
}
|
||||||
$rows = $this->db->fetchAllNumeric($sql . $order . ' LIMIT ' . self::BATCH, $params);
|
$rows = $this->db->fetchAllNumeric($sql . $order . ' LIMIT ' . self::BATCH, $params);
|
||||||
@@ -213,22 +296,29 @@ final class DbDumper {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
yield $rows;
|
yield $rows;
|
||||||
$end = end($rows);
|
|
||||||
$last = array_map(fn ($i) => $end[$i], $pkIdx);
|
|
||||||
if (count($rows) < self::BATCH) {
|
if (count($rows) < self::BATCH) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
$end = end($rows);
|
||||||
|
$last = array_map(fn ($c) => $end[array_search($c, $columns, true)], $pk);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private function insertBatch(string $table, array $rows, array $binary): int {
|
/** One INSERT with several rows (portable multi-row VALUES). */
|
||||||
foreach ($rows as $row) {
|
private function insertBatch(string $table, array $columns, array $rows, array $binary): int {
|
||||||
|
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
||||||
|
$placeholders = '(' . implode(', ', array_fill(0, count($columns), '?')) . ')';
|
||||||
|
$sql = 'INSERT INTO ' . $q($table) . ' (' . implode(', ', array_map($q, $columns)) . ') VALUES '
|
||||||
|
. implode(', ', array_fill(0, count($rows), $placeholders));
|
||||||
|
$params = [];
|
||||||
$types = [];
|
$types = [];
|
||||||
foreach ($row as $col => $_) {
|
foreach ($rows as $row) {
|
||||||
$types[$col] = isset($binary[$col]) ? \Doctrine\DBAL\ParameterType::LARGE_OBJECT : \Doctrine\DBAL\ParameterType::STRING;
|
foreach ($columns as $i => $col) {
|
||||||
|
$params[] = $row[$i];
|
||||||
|
$types[] = isset($binary[$col]) ? \Doctrine\DBAL\ParameterType::LARGE_OBJECT : ($row[$i] === null ? \Doctrine\DBAL\ParameterType::NULL : \Doctrine\DBAL\ParameterType::STRING);
|
||||||
}
|
}
|
||||||
$this->db->insert($this->db->quoteIdentifier($table), array_combine(array_map(fn ($c) => $this->db->quoteIdentifier($c), array_keys($row)), $row), array_values($types));
|
|
||||||
}
|
}
|
||||||
|
$this->db->executeStatement($sql, $params, $types);
|
||||||
return count($rows);
|
return count($rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+72
-7
@@ -11,7 +11,7 @@ use OCA\NgBackup\Repository\IIndexCache;
|
|||||||
use OCP\DB\Exception;
|
use OCP\DB\Exception;
|
||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
|
||||||
class DbIndexCache implements IIndexCache {
|
final class DbIndexCache implements IIndexCache {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private IDBConnection $db,
|
private IDBConnection $db,
|
||||||
) {
|
) {
|
||||||
@@ -30,19 +30,84 @@ class DbIndexCache implements IIndexCache {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function put(string $repositoryId, string $packId, array $entries): void {
|
public function put(string $repositoryId, string $packId, array $entries): void {
|
||||||
|
try {
|
||||||
|
$this->insert($repositoryId, $packId, $entries);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In one transaction: on PostgreSQL with its default settings every statement of its own
|
||||||
|
* waits for a disk sync, which made 27,547 rows take more than 6 minutes. Rows already there
|
||||||
|
* are replaced. Should the transaction fail (another process writing the same rows), the
|
||||||
|
* rows are written one by one, keeping rows that are already there.
|
||||||
|
*/
|
||||||
|
public function putMany(string $repositoryId, array $packs): void {
|
||||||
|
if ($packs === []) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$this->transaction(function () use ($repositoryId, $packs): void {
|
||||||
|
$this->remove($repositoryId, array_keys($packs));
|
||||||
|
$this->insertAll($repositoryId, $packs);
|
||||||
|
});
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
if ($this->db->inTransaction()) {
|
||||||
|
throw $e; // a transaction of the caller's, which may be spoilt now
|
||||||
|
}
|
||||||
|
foreach ($packs as $packId => $entries) {
|
||||||
|
$this->put($repositoryId, $packId, $entries);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** All or nothing: when it fails, the cache stays as it was. */
|
||||||
|
public function replaceAll(string $repositoryId, array $packs): void {
|
||||||
|
$this->transaction(function () use ($repositoryId, $packs): void {
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->delete('ngb_index')->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId)));
|
||||||
|
$qb->executeStatement();
|
||||||
|
$this->insertAll($repositoryId, $packs);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Run $write in a transaction of its own, or as part of the caller's when there is one. */
|
||||||
|
private function transaction(\Closure $write): void {
|
||||||
|
if ($this->db->inTransaction()) {
|
||||||
|
$write();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$this->db->beginTransaction();
|
||||||
|
try {
|
||||||
|
$write();
|
||||||
|
$this->db->commit();
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
if ($this->db->inTransaction()) {
|
||||||
|
$this->db->rollBack();
|
||||||
|
}
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @param array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> $packs */
|
||||||
|
private function insertAll(string $repositoryId, array $packs): void {
|
||||||
|
foreach ($packs as $packId => $entries) {
|
||||||
|
$this->insert($repositoryId, $packId, $entries);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||||
|
private function insert(string $repositoryId, string $packId, array $entries): void {
|
||||||
$qb = $this->db->getQueryBuilder();
|
$qb = $this->db->getQueryBuilder();
|
||||||
$qb->insert('ngb_index')->values([
|
$qb->insert('ngb_index')->values([
|
||||||
'repository_id' => $qb->createNamedParameter($repositoryId),
|
'repository_id' => $qb->createNamedParameter($repositoryId),
|
||||||
'pack_id' => $qb->createNamedParameter($packId),
|
'pack_id' => $qb->createNamedParameter($packId),
|
||||||
'entries' => $qb->createNamedParameter(json_encode($entries, JSON_THROW_ON_ERROR)),
|
'entries' => $qb->createNamedParameter(json_encode($entries, JSON_THROW_ON_ERROR)),
|
||||||
]);
|
]);
|
||||||
try {
|
|
||||||
$qb->executeStatement();
|
$qb->executeStatement();
|
||||||
} catch (Exception $e) {
|
|
||||||
if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function remove(string $repositoryId, array $packIds): void {
|
public function remove(string $repositoryId, array $packIds): void {
|
||||||
|
|||||||
+1
-1
@@ -33,7 +33,7 @@ use OCP\AppFramework\Db\Entity;
|
|||||||
* @method int|null getFinishedAt()
|
* @method int|null getFinishedAt()
|
||||||
* @method void setFinishedAt(?int $v)
|
* @method void setFinishedAt(?int $v)
|
||||||
*/
|
*/
|
||||||
class Run extends Entity {
|
final class Run extends Entity {
|
||||||
public const RUNNING = 'running';
|
public const RUNNING = 'running';
|
||||||
public const DONE = 'done';
|
public const DONE = 'done';
|
||||||
public const FAILED = 'failed';
|
public const FAILED = 'failed';
|
||||||
|
|||||||
+46
-2
@@ -12,11 +12,41 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
|
|||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
|
||||||
/** @template-extends QBMapper<Run> */
|
/** @template-extends QBMapper<Run> */
|
||||||
class RunMapper extends QBMapper {
|
final class RunMapper extends QBMapper {
|
||||||
public function __construct(IDBConnection $db) {
|
public function __construct(IDBConnection $db) {
|
||||||
parent::__construct($db, 'ngb_runs', Run::class);
|
parent::__construct($db, 'ngb_runs', Run::class);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write $run only if it is still running and unchanged since it was loaded (updated_at
|
||||||
|
* $loadedAt), in one statement: two processes that loaded the same run cannot both write it.
|
||||||
|
* The new updated_at is always later than $loadedAt, so every write is seen as a change.
|
||||||
|
*
|
||||||
|
* @return Run|null the written run, or null when another process wrote it first
|
||||||
|
*/
|
||||||
|
public function updateIfUnchanged(Run $run, int $loadedAt): ?Run {
|
||||||
|
$run->setUpdatedAt(max(time(), $loadedAt + 1));
|
||||||
|
$types = $run->getFieldTypes();
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->update($this->getTableName());
|
||||||
|
foreach (array_keys($run->getUpdatedFields()) as $field) {
|
||||||
|
if ($field === 'id') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$value = $run->{'get' . ucfirst($field)}();
|
||||||
|
$type = ($types[$field] ?? 'string') === 'integer' ? IQueryBuilder::PARAM_INT : IQueryBuilder::PARAM_STR;
|
||||||
|
$qb->set($run->propertyToColumn($field), $qb->createNamedParameter($value, $value === null ? IQueryBuilder::PARAM_NULL : $type));
|
||||||
|
}
|
||||||
|
$qb->where($qb->expr()->eq('id', $qb->createNamedParameter($run->getId(), IQueryBuilder::PARAM_INT)))
|
||||||
|
->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Run::RUNNING)))
|
||||||
|
->andWhere($qb->expr()->eq('updated_at', $qb->createNamedParameter($loadedAt, IQueryBuilder::PARAM_INT)));
|
||||||
|
if ($qb->executeStatement() !== 1) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
$run->resetUpdatedFields();
|
||||||
|
return $run;
|
||||||
|
}
|
||||||
|
|
||||||
public function find(int $id): Run {
|
public function find(int $id): Run {
|
||||||
$qb = $this->db->getQueryBuilder();
|
$qb = $this->db->getQueryBuilder();
|
||||||
$qb->select('*')->from($this->getTableName())->where($qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
|
$qb->select('*')->from($this->getTableName())->where($qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
|
||||||
@@ -24,9 +54,12 @@ class RunMapper extends QBMapper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<Run> */
|
/** @return list<Run> */
|
||||||
public function findRunning(?int $targetId = null): array {
|
public function findRunning(?int $targetId = null, ?string $kind = null): array {
|
||||||
$qb = $this->db->getQueryBuilder();
|
$qb = $this->db->getQueryBuilder();
|
||||||
$qb->select('*')->from($this->getTableName())->where($qb->expr()->eq('status', $qb->createNamedParameter(Run::RUNNING)));
|
$qb->select('*')->from($this->getTableName())->where($qb->expr()->eq('status', $qb->createNamedParameter(Run::RUNNING)));
|
||||||
|
if ($kind !== null) {
|
||||||
|
$qb->andWhere($qb->expr()->eq('kind', $qb->createNamedParameter($kind)));
|
||||||
|
}
|
||||||
if ($targetId !== null) {
|
if ($targetId !== null) {
|
||||||
$qb->andWhere($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)));
|
$qb->andWhere($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)));
|
||||||
}
|
}
|
||||||
@@ -40,6 +73,17 @@ class RunMapper extends QBMapper {
|
|||||||
return $this->findEntities($qb);
|
return $this->findEntities($qb);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @return list<Run> runs of $kind started at or after $since, newest first */
|
||||||
|
public function startedSince(int $targetId, string $kind, int $since): array {
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->select('*')->from($this->getTableName())
|
||||||
|
->where($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)))
|
||||||
|
->andWhere($qb->expr()->eq('kind', $qb->createNamedParameter($kind)))
|
||||||
|
->andWhere($qb->expr()->gte('started_at', $qb->createNamedParameter($since, IQueryBuilder::PARAM_INT)))
|
||||||
|
->orderBy('id', 'DESC');
|
||||||
|
return $this->findEntities($qb);
|
||||||
|
}
|
||||||
|
|
||||||
public function lastFinished(int $targetId, string $kind): ?Run {
|
public function lastFinished(int $targetId, string $kind): ?Run {
|
||||||
$qb = $this->db->getQueryBuilder();
|
$qb = $this->db->getQueryBuilder();
|
||||||
$qb->select('*')->from($this->getTableName())
|
$qb->select('*')->from($this->getTableName())
|
||||||
|
|||||||
+1
-1
@@ -25,7 +25,7 @@ use OCP\AppFramework\Db\Entity;
|
|||||||
* @method int|null getBytes()
|
* @method int|null getBytes()
|
||||||
* @method void setBytes(?int $v)
|
* @method void setBytes(?int $v)
|
||||||
*/
|
*/
|
||||||
class Snapshot extends Entity {
|
final class Snapshot extends Entity {
|
||||||
protected int $targetId = 0;
|
protected int $targetId = 0;
|
||||||
protected string $snapshotId = '';
|
protected string $snapshotId = '';
|
||||||
protected string $kind = '';
|
protected string $kind = '';
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
|
|||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
|
||||||
/** @template-extends QBMapper<Snapshot> */
|
/** @template-extends QBMapper<Snapshot> */
|
||||||
class SnapshotMapper extends QBMapper {
|
final class SnapshotMapper extends QBMapper {
|
||||||
public function __construct(IDBConnection $db) {
|
public function __construct(IDBConnection $db) {
|
||||||
parent::__construct($db, 'ngb_snapshots', Snapshot::class);
|
parent::__construct($db, 'ngb_snapshots', Snapshot::class);
|
||||||
}
|
}
|
||||||
|
|||||||
+9
-1
@@ -22,10 +22,14 @@ use OCP\AppFramework\Db\Entity;
|
|||||||
* @method void setBasePath(string $v)
|
* @method void setBasePath(string $v)
|
||||||
* @method string|null getRepositoryId()
|
* @method string|null getRepositoryId()
|
||||||
* @method void setRepositoryId(?string $v)
|
* @method void setRepositoryId(?string $v)
|
||||||
|
* @method bool|null getAppendOnly()
|
||||||
|
* @method void setAppendOnly(bool $v)
|
||||||
|
* @method int|null getMaxBytes()
|
||||||
|
* @method void setMaxBytes(?int $v)
|
||||||
* @method int getCreatedAt()
|
* @method int getCreatedAt()
|
||||||
* @method void setCreatedAt(int $v)
|
* @method void setCreatedAt(int $v)
|
||||||
*/
|
*/
|
||||||
class Target extends Entity {
|
final class Target extends Entity {
|
||||||
protected string $name = '';
|
protected string $name = '';
|
||||||
protected string $backend = '';
|
protected string $backend = '';
|
||||||
protected string $auth = '';
|
protected string $auth = '';
|
||||||
@@ -33,8 +37,12 @@ class Target extends Entity {
|
|||||||
protected string $basePath = '';
|
protected string $basePath = '';
|
||||||
protected ?string $repositoryId = null;
|
protected ?string $repositoryId = null;
|
||||||
protected int $createdAt = 0;
|
protected int $createdAt = 0;
|
||||||
|
protected ?bool $appendOnly = null;
|
||||||
|
protected ?int $maxBytes = null;
|
||||||
|
|
||||||
public function __construct() {
|
public function __construct() {
|
||||||
$this->addType('createdAt', 'integer');
|
$this->addType('createdAt', 'integer');
|
||||||
|
$this->addType('appendOnly', 'boolean');
|
||||||
|
$this->addType('maxBytes', 'integer');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use OCP\AppFramework\Db\QBMapper;
|
|||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
|
||||||
/** @template-extends QBMapper<Target> */
|
/** @template-extends QBMapper<Target> */
|
||||||
class TargetMapper extends QBMapper {
|
final class TargetMapper extends QBMapper {
|
||||||
public function __construct(IDBConnection $db) {
|
public function __construct(IDBConnection $db) {
|
||||||
parent::__construct($db, 'ngb_targets', Target::class);
|
parent::__construct($db, 'ngb_targets', Target::class);
|
||||||
}
|
}
|
||||||
|
|||||||
+104
-12
@@ -49,9 +49,11 @@ final class BackupRun {
|
|||||||
public static function step(Repository $repo, array $state, float $deadline): array {
|
public static function step(Repository $repo, array $state, float $deadline): array {
|
||||||
$state['steps']++;
|
$state['steps']++;
|
||||||
if ($state['phase'] === 'files') {
|
if ($state['phase'] === 'files') {
|
||||||
$walker = new TreeWalker($state['roots'], $state['exclude']);
|
|
||||||
$parent = new ParentCursor($repo, $state['parent'], $state['parentPos']);
|
|
||||||
$stats = $state['stats'];
|
$stats = $state['stats'];
|
||||||
|
$walker = new TreeWalker($state['roots'], $state['exclude'], function (string $logical) use (&$state, &$stats): void {
|
||||||
|
self::recordUnreadable($state, $stats, $logical);
|
||||||
|
});
|
||||||
|
$parent = new ParentCursor($repo, $state['parent'], $state['parentPos']);
|
||||||
$lines = [];
|
$lines = [];
|
||||||
$finishedWalk = true;
|
$finishedWalk = true;
|
||||||
|
|
||||||
@@ -68,15 +70,20 @@ final class BackupRun {
|
|||||||
$stats['reused']++;
|
$stats['reused']++;
|
||||||
} else {
|
} else {
|
||||||
$cur = $state['cur'];
|
$cur = $state['cur'];
|
||||||
if ($cur === null || $cur['path'] !== $logical || $cur['version'] !== $version) {
|
if ($cur !== null && $cur['path'] === $logical && $cur['version'] !== $version) {
|
||||||
if ($cur !== null && $cur['path'] === $logical) {
|
// Changed since the previous step: start over, unless it keeps changing.
|
||||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1; // changed since the previous step
|
if (self::countRetry($state, $stats, $logical)) {
|
||||||
|
$cur['version'] = $version;
|
||||||
|
} else {
|
||||||
|
$cur = null;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if ($cur === null || $cur['path'] !== $logical) {
|
||||||
$cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version];
|
$cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version];
|
||||||
}
|
}
|
||||||
$fh = fopen($path, 'rb');
|
$fh = @fopen($path, 'rb');
|
||||||
if ($fh === false) {
|
if ($fh === false) {
|
||||||
$stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1;
|
self::recordUnreadable($state, $stats, $logical);
|
||||||
$state['cur'] = null;
|
$state['cur'] = null;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -91,19 +98,25 @@ final class BackupRun {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
fclose($fh);
|
fclose($fh);
|
||||||
if (Repository::fileVersion($path) !== $cur['version']) {
|
$now = Repository::fileVersion($path);
|
||||||
|
if ($now !== $cur['version']) {
|
||||||
|
if (!self::countRetry($state, $stats, $logical)) {
|
||||||
// Changed while reading: read it again from the start (now or next step).
|
// Changed while reading: read it again from the start (now or next step).
|
||||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
|
||||||
$state['cur'] = null;
|
$state['cur'] = null;
|
||||||
$finishedWalk = false;
|
$finishedWalk = false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// Keeps changing (a live database, VM image or log): keep what was read,
|
||||||
|
// like any copy of a file in use, and report it instead of retrying forever.
|
||||||
|
$cur['version'] = $now;
|
||||||
|
}
|
||||||
if (!$complete) {
|
if (!$complete) {
|
||||||
$state['cur'] = $cur;
|
$state['cur'] = $cur;
|
||||||
$finishedWalk = false;
|
$finishedWalk = false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
$blobs = $cur['blobs'];
|
$blobs = $cur['blobs'];
|
||||||
|
$size = $cur['offset'];
|
||||||
$state['cur'] = null;
|
$state['cur'] = null;
|
||||||
$stats['read']++;
|
$stats['read']++;
|
||||||
}
|
}
|
||||||
@@ -133,19 +146,98 @@ final class BackupRun {
|
|||||||
|
|
||||||
if ($state['phase'] === 'finish') {
|
if ($state['phase'] === 'finish') {
|
||||||
$segments = $state['segments'];
|
$segments = $state['segments'];
|
||||||
$repo->writeSnapshot($state['snapshot'],
|
$repo->putSnapshot($state['snapshot'],
|
||||||
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
|
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
|
||||||
'roots' => array_keys($state['roots']), 'stats' => $state['stats']] + $state['meta'],
|
'roots' => array_keys($state['roots']), 'stats' => $state['stats'],
|
||||||
|
'warnings' => self::warnings($state)] + $state['meta'],
|
||||||
(function () use ($repo, $segments) {
|
(function () use ($repo, $segments) {
|
||||||
foreach ($segments as $blobs) {
|
foreach ($segments as $blobs) {
|
||||||
yield from $repo->readLines($blobs);
|
yield from $repo->readLines($blobs);
|
||||||
}
|
}
|
||||||
})());
|
})());
|
||||||
$state['phase'] = 'done';
|
$state['phase'] = 'commit';
|
||||||
}
|
}
|
||||||
return $state;
|
return $state;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Progress of the files phase as a fraction (at most 0.99 until it is done), or null without an
|
||||||
|
* estimate. Bytes of a large file that is still being read count too.
|
||||||
|
*
|
||||||
|
* @return array{fraction: float, approximate: bool}|null
|
||||||
|
*/
|
||||||
|
public static function progress(array $state): ?array {
|
||||||
|
$estimate = $state['estimate'] ?? null;
|
||||||
|
if (($state['phase'] ?? null) !== 'files' || !is_array($estimate) || ($estimate['bytes'] ?? 0) <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
$done = (int)($state['stats']['bytes'] ?? 0) + (int)($state['cur']['offset'] ?? 0);
|
||||||
|
return ['fraction' => min(0.99, $done / (int)$estimate['bytes']), 'approximate' => (bool)($estimate['approximate'] ?? true)];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* progress() of a run, while it is in its files phase.
|
||||||
|
*
|
||||||
|
* @return array{fraction: float, approximate: bool}|null
|
||||||
|
*/
|
||||||
|
public static function progressOf(\OCA\NgBackup\Db\Run $run): ?array {
|
||||||
|
if ($run->getStatus() !== \OCA\NgBackup\Db\Run::RUNNING || $run->getPhase() !== 'files') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
} catch (\JsonException) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return is_array($state['files'] ?? null) ? self::progress($state['files']) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A file or directory that could not be read: it is missing from this snapshot. Remembered (first
|
||||||
|
* 1000, deduplicated across steps) so the run can report it instead of silently succeeding.
|
||||||
|
*/
|
||||||
|
private static function recordUnreadable(array &$state, array &$stats, string $logical): void {
|
||||||
|
if (isset($state['unreadable'][$logical])) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (count($state['unreadable'] ?? []) < 1000) {
|
||||||
|
$state['unreadable'][$logical] = true;
|
||||||
|
}
|
||||||
|
$stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return array{unreadable: list<string>, changing: list<string>} examples for the report (first 50 each) */
|
||||||
|
public static function warnings(array $state): array {
|
||||||
|
return [
|
||||||
|
'unreadable' => array_slice(array_map('strval', array_keys($state['unreadable'] ?? [])), 0, 50),
|
||||||
|
'changing' => array_slice(array_map('strval', array_values($state['changing'] ?? [])), 0, 50),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Retries of one file that changes while it is read, before its last read is kept anyway. */
|
||||||
|
public const MAX_RETRIES = 3;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count a retry of $logical. Returns true once it has changed MAX_RETRIES times: the caller then
|
||||||
|
* keeps the data it read, and the file is listed in $state['changing'] (first 50) for the report.
|
||||||
|
*/
|
||||||
|
private static function countRetry(array &$state, array &$stats, string $logical): bool {
|
||||||
|
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
||||||
|
$n = (($state['retryPath'] ?? null) === $logical ? (int)($state['retryCount'] ?? 0) : 0) + 1;
|
||||||
|
$state['retryPath'] = $logical;
|
||||||
|
$state['retryCount'] = $n;
|
||||||
|
if ($n < self::MAX_RETRIES) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if ($n === self::MAX_RETRIES) {
|
||||||
|
$stats['changing'] = ($stats['changing'] ?? 0) + 1;
|
||||||
|
if (count($state['changing'] ?? []) < 50) {
|
||||||
|
$state['changing'][] = $logical;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
private static function readFull($fh, int $length): string {
|
private static function readFull($fh, int $length): string {
|
||||||
$buffer = '';
|
$buffer = '';
|
||||||
while (strlen($buffer) < $length && !feof($fh)) {
|
while (strlen($buffer) < $length && !feof($fh)) {
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ final class TreeWalker {
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private array $roots,
|
private array $roots,
|
||||||
private array $exclude = [],
|
private array $exclude = [],
|
||||||
|
/** Called with the logical path of a directory that cannot be listed. */
|
||||||
|
private ?\Closure $onUnreadable = null,
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -57,6 +59,9 @@ final class TreeWalker {
|
|||||||
}
|
}
|
||||||
$names = @scandir($abs);
|
$names = @scandir($abs);
|
||||||
if ($names === false) {
|
if ($names === false) {
|
||||||
|
if ($this->onUnreadable !== null) {
|
||||||
|
($this->onUnreadable)($logical === '' ? '/' : $logical);
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
$names = array_values(array_diff($names, ['.', '..']));
|
$names = array_values(array_diff($names, ['.', '..']));
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ use OCP\DB\Types;
|
|||||||
use OCP\Migration\IOutput;
|
use OCP\Migration\IOutput;
|
||||||
use OCP\Migration\SimpleMigrationStep;
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
class Version000001Date20261002000000 extends SimpleMigrationStep {
|
final class Version000001Date20261002000000 extends SimpleMigrationStep {
|
||||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||||
/** @var ISchemaWrapper $schema */
|
/** @var ISchemaWrapper $schema */
|
||||||
$schema = $schemaClosure();
|
$schema = $schemaClosure();
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use OCP\Migration\IOutput;
|
|||||||
use OCP\Migration\SimpleMigrationStep;
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
/** Local cache of the per-pack index files, so opening a repository does not download them all. */
|
/** Local cache of the per-pack index files, so opening a repository does not download them all. */
|
||||||
class Version000002Date20261002010000 extends SimpleMigrationStep {
|
final class Version000002Date20261002010000 extends SimpleMigrationStep {
|
||||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||||
/** @var ISchemaWrapper $schema */
|
/** @var ISchemaWrapper $schema */
|
||||||
$schema = $schemaClosure();
|
$schema = $schemaClosure();
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Migration;
|
||||||
|
|
||||||
|
use Closure;
|
||||||
|
use OCP\DB\ISchemaWrapper;
|
||||||
|
use OCP\DB\Types;
|
||||||
|
use OCP\Migration\IOutput;
|
||||||
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
|
/** Append-only locations: NG Backup never deletes anything there (cleanup by the location's own lifecycle rules). */
|
||||||
|
final class Version000003Date20261002020000 extends SimpleMigrationStep {
|
||||||
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||||
|
/** @var ISchemaWrapper $schema */
|
||||||
|
$schema = $schemaClosure();
|
||||||
|
$t = $schema->getTable('ngb_targets');
|
||||||
|
if (!$t->hasColumn('append_only')) {
|
||||||
|
$t->addColumn('append_only', Types::BOOLEAN, ['notnull' => false, 'default' => false]);
|
||||||
|
return $schema;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Migration;
|
||||||
|
|
||||||
|
use Closure;
|
||||||
|
use OCP\DB\ISchemaWrapper;
|
||||||
|
use OCP\DB\Types;
|
||||||
|
use OCP\Migration\IOutput;
|
||||||
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
|
/** Optional size limit per location (cost control; S3 has no bucket quota). */
|
||||||
|
final class Version000004Date20261002030000 extends SimpleMigrationStep {
|
||||||
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||||
|
/** @var ISchemaWrapper $schema */
|
||||||
|
$schema = $schemaClosure();
|
||||||
|
$t = $schema->getTable('ngb_targets');
|
||||||
|
if (!$t->hasColumn('max_bytes')) {
|
||||||
|
$t->addColumn('max_bytes', Types::BIGINT, ['notnull' => false, 'unsigned' => true]);
|
||||||
|
return $schema;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Migration;
|
||||||
|
|
||||||
|
use Closure;
|
||||||
|
use OCP\DB\ISchemaWrapper;
|
||||||
|
use OCP\DB\Types;
|
||||||
|
use OCP\Migration\IOutput;
|
||||||
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
|
/** Leases with an expiry: a killed process cannot block a location for longer than its lease. */
|
||||||
|
final class Version000005Date20261002040000 extends SimpleMigrationStep {
|
||||||
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||||
|
/** @var ISchemaWrapper $schema */
|
||||||
|
$schema = $schemaClosure();
|
||||||
|
if ($schema->hasTable('ngb_leases')) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
$t = $schema->createTable('ngb_leases');
|
||||||
|
$t->addColumn('id', Types::STRING, ['notnull' => true, 'length' => 32]);
|
||||||
|
$t->addColumn('name', Types::STRING, ['notnull' => true, 'length' => 128]);
|
||||||
|
$t->addColumn('mode', Types::STRING, ['notnull' => true, 'length' => 1]);
|
||||||
|
$t->addColumn('expires_at', Types::BIGINT, ['notnull' => true, 'unsigned' => true]);
|
||||||
|
$t->setPrimaryKey(['id']);
|
||||||
|
$t->addIndex(['name'], 'ngb_leases_name');
|
||||||
|
return $schema;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Notification;
|
||||||
|
|
||||||
|
use OCA\NgBackup\AppInfo\Application;
|
||||||
|
use OCP\IURLGenerator;
|
||||||
|
use OCP\L10N\IFactory;
|
||||||
|
use OCP\Notification\INotification;
|
||||||
|
use OCP\Notification\INotifier;
|
||||||
|
use OCP\Notification\UnknownNotificationException;
|
||||||
|
|
||||||
|
final class Notifier implements INotifier {
|
||||||
|
public function __construct(
|
||||||
|
private IFactory $l10n,
|
||||||
|
private IURLGenerator $url,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getID(): string {
|
||||||
|
return Application::APP_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getName(): string {
|
||||||
|
return $this->l10n->get(Application::APP_ID)->t('NG Backup');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function prepare(INotification $notification, string $languageCode): INotification {
|
||||||
|
if ($notification->getApp() !== Application::APP_ID) {
|
||||||
|
throw new UnknownNotificationException();
|
||||||
|
}
|
||||||
|
$l = $this->l10n->get(Application::APP_ID, $languageCode);
|
||||||
|
$p = $notification->getSubjectParameters();
|
||||||
|
switch ($notification->getSubject()) {
|
||||||
|
case 'run_failed':
|
||||||
|
$notification->setParsedSubject($p['kind'] === 'restore'
|
||||||
|
? $l->t('Restore from %s failed', [$p['target']])
|
||||||
|
: $l->t('Backup to %s failed', [$p['target']]));
|
||||||
|
$notification->setParsedMessage((string)($p['error'] ?? ''));
|
||||||
|
break;
|
||||||
|
case 'run_warnings':
|
||||||
|
$notification->setParsedSubject($l->t('Backup to %1$s finished with warnings: %2$s unreadable, %3$s changed while being read', [$p['target'], $p['unreadable'], $p['changing']]));
|
||||||
|
$notification->setParsedMessage($l->t('Unreadable files and folders are missing from this backup (check their permissions). Files that kept changing were saved as they were while being read. Examples: %s', [$p['examples'] ?? '']));
|
||||||
|
break;
|
||||||
|
case 'quota_warning':
|
||||||
|
$notification->setParsedSubject($l->t('Backup location %1$s is at %2$s%% of its limit (%3$s of %4$s)', [$p['target'], $p['percent'], $p['used'], $p['limit']]));
|
||||||
|
$notification->setParsedMessage($l->t('Backups stop when the limit is reached. Raise the limit, or let the retention policy remove older backups.'));
|
||||||
|
break;
|
||||||
|
case 'backup_stale':
|
||||||
|
// 'never' was stored by 0.9.4 and earlier
|
||||||
|
$notification->setParsedSubject(in_array($p['since'] ?? '', ['', 'never'], true)
|
||||||
|
? $l->t('No successful backup to %s yet', [$p['target']])
|
||||||
|
: $l->t('No successful backup to %1$s since %2$s', [$p['target'], $p['since']]));
|
||||||
|
$notification->setParsedMessage($l->t('Check the backup settings and the server log.'));
|
||||||
|
break;
|
||||||
|
case 'security_event':
|
||||||
|
$who = (string)($p['actor'] ?? '');
|
||||||
|
$notification->setParsedSubject(match ($p['event'] ?? '') {
|
||||||
|
'kit_downloaded' => $l->t('%s downloaded the backup recovery kit', [$who]),
|
||||||
|
'target_removed' => $l->t('%1$s removed backup location %2$s', [$who, $p['target'] ?? '']),
|
||||||
|
'target_changed' => $l->t('%1$s changed the connection settings of backup location %2$s', [$who, $p['target'] ?? '']),
|
||||||
|
'append_only_off' => $l->t('%1$s turned off append-only for backup location %2$s', [$who, $p['target'] ?? '']),
|
||||||
|
'retention_changed' => $l->t('%1$s changed the backup retention to %2$s', [$who, $p['policy'] ?? '']),
|
||||||
|
'snapshots_forgotten' => $l->t('%1$s moved %2$s snapshots of %3$s to the trash; they are deleted permanently after %4$s unless restored', [$who, $p['count'] ?? '', $p['target'] ?? '', $p['until'] ?? '']),
|
||||||
|
'snapshot_untrashed' => $l->t('%1$s restored snapshot %2$s of %3$s from the trash', [$who, $p['snapshot'] ?? '', $p['target'] ?? '']),
|
||||||
|
'catalog_trusted' => $l->t('%1$s accepted an older state of backup location %2$s (catalog generation %3$s)', [$who, $p['target'] ?? '', $p['generation'] ?? '']),
|
||||||
|
'slot_added' => $l->t('%1$s added backup passphrase slot %2$s (%3$s); download and confirm the new recovery kit', [$who, $p['slot'] ?? '', $p['label'] ?? '']),
|
||||||
|
'slot_replaced' => $l->t('%1$s set a new passphrase for backup key slot %2$s (%3$s); download and confirm the new recovery kit', [$who, $p['slot'] ?? '', $p['label'] ?? '']),
|
||||||
|
'user_export_retention_changed' => $l->t('%1$s changed how many exports per user are kept to %2$s', [$who, $p['keep'] ?? '']),
|
||||||
|
'user_exports_forgotten' => $l->t('%1$s removed %2$s per-user exports from %3$s', [$who, $p['count'] ?? '', $p['target'] ?? '']),
|
||||||
|
'slot_removed' => $l->t('%1$s removed backup passphrase slot %2$s; download and confirm the new recovery kit', [$who, $p['slot'] ?? '']),
|
||||||
|
default => $l->t('NG Backup security event'),
|
||||||
|
});
|
||||||
|
$notification->setParsedMessage($l->t('If this was not you or another administrator, check who has admin access.'));
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new UnknownNotificationException();
|
||||||
|
}
|
||||||
|
$notification->setLink($this->url->linkToRouteAbsolute('settings.AdminSettings.index', ['section' => 'ng_backup']));
|
||||||
|
$notification->setIcon($this->url->getAbsoluteURL($this->url->imagePath(Application::APP_ID, 'app-dark.svg')));
|
||||||
|
return $notification;
|
||||||
|
}
|
||||||
|
}
|
||||||
+138
-13
@@ -25,7 +25,11 @@ final class BlobIndex {
|
|||||||
private array $pending = [];
|
private array $pending = [];
|
||||||
|
|
||||||
private ?IIndexCache $cache = null;
|
private ?IIndexCache $cache = null;
|
||||||
|
/** @var list<\Closure():void>|null cache writes held back by holdCache(), or null when writing directly */
|
||||||
|
private ?array $heldCacheWrites = null;
|
||||||
private string $repositoryId = '';
|
private string $repositoryId = '';
|
||||||
|
/** Index files written to the cache per database transaction while loading. */
|
||||||
|
private const CACHE_LOT = 256;
|
||||||
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
|
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
|
||||||
public int $downloaded = 0;
|
public int $downloaded = 0;
|
||||||
|
|
||||||
@@ -40,30 +44,106 @@ final class BlobIndex {
|
|||||||
$this->repositoryId = $repositoryId;
|
$this->repositoryId = $repositoryId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hold back cache writes until releaseCache(). The database dump reads inside a read-only
|
||||||
|
* transaction on the same connection that the cache writes to: a write there fails on
|
||||||
|
* PostgreSQL and would be rolled back with the dump transaction elsewhere.
|
||||||
|
*/
|
||||||
|
public function holdCache(): void {
|
||||||
|
$this->heldCacheWrites ??= [];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Write the cache changes held back since holdCache(). */
|
||||||
|
public function releaseCache(): void {
|
||||||
|
$writes = $this->heldCacheWrites ?? [];
|
||||||
|
$this->heldCacheWrites = null;
|
||||||
|
foreach ($writes as $write) {
|
||||||
|
$write();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||||
|
private function cachePut(string $packId, array $entries): void {
|
||||||
|
if ($this->heldCacheWrites !== null) {
|
||||||
|
$this->heldCacheWrites[] = function () use ($packId, $entries): void {
|
||||||
|
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||||
|
};
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @param list<string> $packIds */
|
||||||
|
private function cacheRemove(array $packIds): void {
|
||||||
|
if ($this->heldCacheWrites !== null) {
|
||||||
|
$this->heldCacheWrites[] = function () use ($packIds): void {
|
||||||
|
$this->cache?->remove($this->repositoryId, $packIds);
|
||||||
|
};
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$this->cache?->remove($this->repositoryId, $packIds);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load the index: one listing of index/, entries from the local cache where available,
|
* Load the index: one listing of index/, entries from the local cache where available,
|
||||||
* and only the index files that are not cached yet are downloaded. Cached packs that no
|
* and only the index files that are not cached yet are downloaded. Cached packs that no
|
||||||
* longer exist on the backend are dropped from the cache.
|
* longer exist on the backend are dropped from the cache.
|
||||||
*/
|
*/
|
||||||
public function load(): void {
|
public function load(bool $verify = false, ?callable $progress = null): void {
|
||||||
$cached = $this->cache?->all($this->repositoryId) ?? [];
|
$this->blobs = [];
|
||||||
|
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
|
||||||
$present = [];
|
$present = [];
|
||||||
$this->downloaded = 0;
|
$this->downloaded = 0;
|
||||||
foreach ($this->backend->list('index') as $path) {
|
$order = []; // pack ids in listing order: where a blob is in two packs, the last listed wins
|
||||||
|
$fetch = [];
|
||||||
|
foreach ($this->backend->listFiles('index') as $path) {
|
||||||
$packId = basename($path);
|
$packId = basename($path);
|
||||||
$present[$packId] = true;
|
$present[$packId] = true;
|
||||||
if (isset($cached[$packId])) {
|
$order[] = $packId;
|
||||||
$entries = $cached[$packId];
|
if (!isset($cached[$packId])) {
|
||||||
} else {
|
$fetch[] = $path;
|
||||||
$fh = $this->backend->get($path);
|
}
|
||||||
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
}
|
||||||
fclose($fh);
|
// The rest is downloaded several at a time where the location allows it: on a large
|
||||||
|
// repository these are tens of thousands of small files. getMany() keeps their order, so
|
||||||
|
// the cached packs listed before each one are added first.
|
||||||
|
$next = 0;
|
||||||
|
$total = count($fetch);
|
||||||
|
$toCache = []; // written to the cache a few hundred at a time, each lot in one transaction
|
||||||
|
foreach ($this->backend->getMany($fetch) as $path => $data) {
|
||||||
|
$packId = basename($path);
|
||||||
|
$entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
||||||
$this->downloaded++;
|
$this->downloaded++;
|
||||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
if ($this->cache !== null) {
|
||||||
|
$toCache[$packId] = $entries;
|
||||||
|
// a check writes the whole cache afresh at the end
|
||||||
|
if (!$verify && count($toCache) >= self::CACHE_LOT) {
|
||||||
|
$this->cache->putMany($this->repositoryId, $toCache);
|
||||||
|
$toCache = [];
|
||||||
}
|
}
|
||||||
foreach ($entries as $e) {
|
|
||||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
|
||||||
}
|
}
|
||||||
|
for (; $next < count($order) && $order[$next] !== $packId; $next++) {
|
||||||
|
$this->add($order[$next], $cached[$order[$next]] ?? []);
|
||||||
|
}
|
||||||
|
$this->add($packId, $entries);
|
||||||
|
$next++;
|
||||||
|
if ($progress !== null) {
|
||||||
|
$progress($this->downloaded, $total);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (; $next < count($order); $next++) {
|
||||||
|
$this->add($order[$next], $cached[$order[$next]] ?? []);
|
||||||
|
}
|
||||||
|
if ($verify && $this->cache !== null) {
|
||||||
|
// Refresh the cache from what was just verified, as the index files list it: a blob in
|
||||||
|
// two packs is in both rows, so that removing one of the packs leaves the other right.
|
||||||
|
// In one transaction: one row at a time took more than 6 minutes on a 1 TB repository
|
||||||
|
// (27,547 packs), long enough for the location to drop the connection.
|
||||||
|
$this->cache->replaceAll($this->repositoryId, $toCache);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ($toCache !== []) {
|
||||||
|
$this->cache?->putMany($this->repositoryId, $toCache);
|
||||||
}
|
}
|
||||||
$stale = array_keys(array_diff_key($cached, $present));
|
$stale = array_keys(array_diff_key($cached, $present));
|
||||||
if ($stale !== []) {
|
if ($stale !== []) {
|
||||||
@@ -71,6 +151,13 @@ final class BlobIndex {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||||
|
private function add(string $packId, array $entries): void {
|
||||||
|
foreach ($entries as $e) {
|
||||||
|
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public function has(string $id): bool {
|
public function has(string $id): bool {
|
||||||
return isset($this->blobs[$id]) || isset($this->pending[$id]);
|
return isset($this->blobs[$id]) || isset($this->pending[$id]);
|
||||||
}
|
}
|
||||||
@@ -97,13 +184,51 @@ final class BlobIndex {
|
|||||||
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
|
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
|
||||||
rewind($stream);
|
rewind($stream);
|
||||||
$this->backend->put('index/' . $packId, $stream);
|
$this->backend->put('index/' . $packId, $stream);
|
||||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
$this->cachePut($packId, $entries);
|
||||||
foreach ($entries as $e) {
|
foreach ($entries as $e) {
|
||||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||||
unset($this->pending[$e['id']]);
|
unset($this->pending[$e['id']]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @return array<string, list<array{id:string, offset:int, length:int}>> packId => blobs currently located in that pack */
|
||||||
|
public function packs(): array {
|
||||||
|
$packs = [];
|
||||||
|
foreach ($this->blobs as $id => [$pack, $offset, $length]) {
|
||||||
|
$packs[$pack][] = ['id' => $id, 'offset' => $offset, 'length' => $length];
|
||||||
|
}
|
||||||
|
return $packs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Forget a pack: delete its index file and cache row. Blobs that now live elsewhere are kept. */
|
||||||
|
public function removePack(string $packId): void {
|
||||||
|
$this->backend->delete('index/' . $packId);
|
||||||
|
$this->cacheRemove([$packId]);
|
||||||
|
foreach ($this->blobs as $id => $loc) {
|
||||||
|
if ($loc[0] === $packId) {
|
||||||
|
unset($this->blobs[$id]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Number of packs that hold blobs of the index. */
|
||||||
|
public function packCount(): int {
|
||||||
|
$packs = [];
|
||||||
|
foreach ($this->blobs as [$pack]) {
|
||||||
|
$packs[$pack] = true;
|
||||||
|
}
|
||||||
|
return count($packs);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Bytes of all blobs in the index (encrypted size, i.e. what is stored in packs). */
|
||||||
|
public function storedBytes(): int {
|
||||||
|
$total = 0;
|
||||||
|
foreach ($this->blobs as [, , $length]) {
|
||||||
|
$total += $length;
|
||||||
|
}
|
||||||
|
return $total;
|
||||||
|
}
|
||||||
|
|
||||||
public function count(): int {
|
public function count(): int {
|
||||||
return count($this->blobs);
|
return count($this->blobs);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Repository;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Backend\IBackend;
|
||||||
|
use OCA\NgBackup\Crypto\StreamCipher;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Chained catalog of a repository's state, against rollback and hidden snapshots.
|
||||||
|
*
|
||||||
|
* Every change to the set of snapshots or trash entries writes catalog/<generation>: the
|
||||||
|
* encrypted list of snapshot and trash ids plus the hash of the previous generation. The server
|
||||||
|
* keeps the newest generation it has seen (ICatalogAnchor). On open, the newest catalog on the
|
||||||
|
* location must be at least the anchored generation and lead back to it through the chain;
|
||||||
|
* snapshots listed in the catalog must exist. Otherwise the location was rolled back or
|
||||||
|
* tampered with, and the repository is refused.
|
||||||
|
*/
|
||||||
|
final class Catalog {
|
||||||
|
private const KEEP = 200; // generations kept on the location (enough to verify long gaps)
|
||||||
|
|
||||||
|
public int $gen = 0;
|
||||||
|
/** @var array<string, true> */
|
||||||
|
public array $snapshots = [];
|
||||||
|
/** @var array<string, true> */
|
||||||
|
public array $trash = [];
|
||||||
|
/** @var array<string, true> manifest path => true, for a per-user export (users/<uid>/<exportId>) */
|
||||||
|
public array $userExports = [];
|
||||||
|
public string $hash = '';
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private IBackend $backend,
|
||||||
|
private StreamCipher $cipher,
|
||||||
|
private string $repositoryId,
|
||||||
|
private ?ICatalogAnchor $anchor,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Load and verify the newest catalog; create the first one for repositories that have none. */
|
||||||
|
public function load(callable $listSnapshots, callable $listTrash, callable $listUserExports): void {
|
||||||
|
$gens = $this->generations();
|
||||||
|
if ($gens === []) {
|
||||||
|
// First use (new or pre-catalog repository): adopt the current state as generation 1.
|
||||||
|
$anchored = $this->anchor?->get($this->repositoryId);
|
||||||
|
if ($anchored !== null) {
|
||||||
|
throw new RepositoryException('The catalog of this repository has disappeared from the location (possible tampering)');
|
||||||
|
}
|
||||||
|
$this->snapshots = array_fill_keys($listSnapshots(), true);
|
||||||
|
$this->trash = array_fill_keys($listTrash(), true);
|
||||||
|
$this->userExports = array_fill_keys($listUserExports(), true);
|
||||||
|
$this->write();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$head = max($gens);
|
||||||
|
[$data, $hash] = $this->read($head);
|
||||||
|
$anchored = $this->anchor?->get($this->repositoryId);
|
||||||
|
if ($anchored !== null) {
|
||||||
|
if ($head < $anchored['gen']) {
|
||||||
|
throw new RepositoryException(sprintf('The location shows catalog generation %d, but this server has seen %d: the backups were rolled back', $head, $anchored['gen']));
|
||||||
|
}
|
||||||
|
// Walk back to the anchored generation, checking every link.
|
||||||
|
[$cur, $curHash] = [$data, $hash];
|
||||||
|
for ($g = $head; $g > $anchored['gen']; $g--) {
|
||||||
|
[$prev, $prevHash] = $this->read($g - 1);
|
||||||
|
if (!hash_equals($prevHash, (string)$cur['prev'])) {
|
||||||
|
throw new RepositoryException("Catalog chain broken at generation $g (possible tampering)");
|
||||||
|
}
|
||||||
|
[$cur, $curHash] = [$prev, $prevHash];
|
||||||
|
}
|
||||||
|
if (!hash_equals($anchored['hash'], $curHash)) {
|
||||||
|
throw new RepositoryException('The catalog does not match what this server saw before (possible tampering)');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$this->gen = $head;
|
||||||
|
$this->hash = $hash;
|
||||||
|
$this->snapshots = array_fill_keys($data['snapshots'], true);
|
||||||
|
$this->trash = array_fill_keys($data['trash'], true);
|
||||||
|
// Catalogs written before user exports existed have no such key: default to empty rather
|
||||||
|
// than adopting the location's current users/ listing (that would skip verification).
|
||||||
|
$this->userExports = array_fill_keys($data['userExports'] ?? [], true);
|
||||||
|
$this->anchor?->set($this->repositoryId, $this->gen, $this->hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Write the next generation after the in-memory state was changed. */
|
||||||
|
public function write(): void {
|
||||||
|
$next = $this->gen + 1;
|
||||||
|
// Two writers starting from the same generation would otherwise overwrite each other
|
||||||
|
// silently and break the chain for whichever server anchored the lost write.
|
||||||
|
if ($this->backend->exists($this->path($next))) {
|
||||||
|
throw new RepositoryException("Catalog generation $next was written by another process at the same time; try again");
|
||||||
|
}
|
||||||
|
$plain = json_encode(['repository' => $this->repositoryId, 'gen' => $next, 'time' => gmdate('c'), 'prev' => $this->hash,
|
||||||
|
'snapshots' => array_keys($this->snapshots), 'trash' => array_keys($this->trash),
|
||||||
|
'userExports' => array_keys($this->userExports)], JSON_THROW_ON_ERROR);
|
||||||
|
$encrypted = $this->cipher->encryptString($plain, $this->ad($next));
|
||||||
|
$stream = fopen('php://memory', 'w+b');
|
||||||
|
fwrite($stream, $encrypted);
|
||||||
|
rewind($stream);
|
||||||
|
$this->backend->put($this->path($next), $stream);
|
||||||
|
$this->gen = $next;
|
||||||
|
$this->hash = hash('sha256', $encrypted);
|
||||||
|
$this->anchor?->set($this->repositoryId, $this->gen, $this->hash);
|
||||||
|
$old = $next - self::KEEP;
|
||||||
|
if ($old > 0) {
|
||||||
|
$this->backend->delete($this->path($old));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return array{0: array, 1: string} decoded catalog and the hash of its encrypted bytes */
|
||||||
|
private function read(int $gen): array {
|
||||||
|
try {
|
||||||
|
$fh = $this->backend->get($this->path($gen));
|
||||||
|
} catch (\Throwable) {
|
||||||
|
throw new RepositoryException("Catalog generation $gen is missing (possible tampering)");
|
||||||
|
}
|
||||||
|
$encrypted = (string)stream_get_contents($fh);
|
||||||
|
fclose($fh);
|
||||||
|
$data = json_decode($this->cipher->decryptString($encrypted, $this->ad($gen)), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
if (($data['repository'] ?? '') !== $this->repositoryId || ($data['gen'] ?? 0) !== $gen) {
|
||||||
|
throw new RepositoryException("Catalog generation $gen belongs to another repository or position");
|
||||||
|
}
|
||||||
|
return [$data, hash('sha256', $encrypted)];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return list<int> */
|
||||||
|
private function generations(): array {
|
||||||
|
return array_map(fn ($p) => (int)basename($p), $this->backend->listFiles('catalog'));
|
||||||
|
}
|
||||||
|
|
||||||
|
private function path(int $gen): string {
|
||||||
|
return sprintf('catalog/%010d', $gen);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function ad(int $gen): string {
|
||||||
|
return 'catalog:' . $this->repositoryId . ':' . $gen;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Repository;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where the server remembers the newest catalog generation it has seen for a repository, outside
|
||||||
|
* the (untrusted) location. A location that presents an older generation, or a chain that does
|
||||||
|
* not lead back to this anchor, is rolled back or tampered with.
|
||||||
|
*/
|
||||||
|
interface ICatalogAnchor {
|
||||||
|
/** @return array{gen:int, hash:string}|null */
|
||||||
|
public function get(string $repositoryId): ?array;
|
||||||
|
|
||||||
|
public function set(string $repositoryId, int $gen, string $hash): void;
|
||||||
|
}
|
||||||
@@ -15,6 +15,20 @@ interface IIndexCache {
|
|||||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||||
public function put(string $repositoryId, string $packId, array $entries): void;
|
public function put(string $repositoryId, string $packId, array $entries): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* put() for many packs at once (one database transaction).
|
||||||
|
*
|
||||||
|
* @param array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> $packs packId => entries
|
||||||
|
*/
|
||||||
|
public function putMany(string $repositoryId, array $packs): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Replace everything cached for the repository with $packs (one database transaction).
|
||||||
|
*
|
||||||
|
* @param array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> $packs packId => entries
|
||||||
|
*/
|
||||||
|
public function replaceAll(string $repositoryId, array $packs): void;
|
||||||
|
|
||||||
/** @param list<string> $packIds */
|
/** @param list<string> $packIds */
|
||||||
public function remove(string $repositoryId, array $packIds): void;
|
public function remove(string $repositoryId, array $packIds): void;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Repository;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Backend\IBackend;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads blobs from packs. Restores and deep verifies read the blobs of a snapshot in the order
|
||||||
|
* they were written, so the stream of the pack read last is kept open and read on from where it
|
||||||
|
* is: one request per pack instead of one per blob (a round trip each, and on most remote
|
||||||
|
* storages a new connection or file handle too). Anything else opens the pack at the offset,
|
||||||
|
* as a single read always did.
|
||||||
|
*
|
||||||
|
* The open stream must not overlap other requests on the location (SFTP cannot read and write
|
||||||
|
* on one connection at once): Repository reaches the location through a PackReaderGuard, which
|
||||||
|
* closes it first.
|
||||||
|
*/
|
||||||
|
final class PackReader {
|
||||||
|
/** A gap up to this size in the open stream is read past instead of opening the pack again. */
|
||||||
|
private const SKIP_MAX = 8 * 1048576;
|
||||||
|
private const CHUNK = 1048576;
|
||||||
|
|
||||||
|
/** @var resource|null */
|
||||||
|
private $stream = null;
|
||||||
|
private string $pack = '';
|
||||||
|
private int $pos = 0;
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private IBackend $backend,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public function read(string $pack, int $offset, int $length): string {
|
||||||
|
if ($this->stream !== null && $this->pack === $pack && $offset >= $this->pos && $offset - $this->pos <= self::SKIP_MAX) {
|
||||||
|
try {
|
||||||
|
$this->skipTo($offset);
|
||||||
|
return $this->readExactly($length);
|
||||||
|
} catch (\Exception) {
|
||||||
|
// The open stream broke (e.g. a timeout); open the pack again below.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$this->close();
|
||||||
|
$fh = $this->backend->get('packs/' . substr($pack, 0, 2) . '/' . $pack);
|
||||||
|
$this->stream = $fh;
|
||||||
|
$this->pack = $pack;
|
||||||
|
$this->pos = 0;
|
||||||
|
try {
|
||||||
|
if ($offset > 0 && (stream_get_meta_data($fh)['seekable'] ?? false) && fseek($fh, $offset) === 0) {
|
||||||
|
$this->pos = $offset;
|
||||||
|
}
|
||||||
|
$this->skipTo($offset); // a stream that cannot seek is read up to the offset
|
||||||
|
return $this->readExactly($length);
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
$this->close();
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function close(): void {
|
||||||
|
$fh = $this->stream;
|
||||||
|
$this->stream = null;
|
||||||
|
if ($fh !== null) {
|
||||||
|
@fclose($fh);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function __destruct() {
|
||||||
|
$this->close();
|
||||||
|
}
|
||||||
|
|
||||||
|
private function skipTo(int $offset): void {
|
||||||
|
while ($this->pos < $offset) {
|
||||||
|
$this->readExactly(min($offset - $this->pos, self::CHUNK));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read $length bytes from the open stream; a short read closes it. */
|
||||||
|
private function readExactly(int $length): string {
|
||||||
|
$fh = $this->stream ?? throw new RepositoryException('No open pack stream');
|
||||||
|
$data = '';
|
||||||
|
while (strlen($data) < $length) {
|
||||||
|
$chunk = fread($fh, min($length - strlen($data), self::CHUNK));
|
||||||
|
if ($chunk === false || $chunk === '') {
|
||||||
|
$this->close();
|
||||||
|
throw new RepositoryException('Short read in pack ' . $this->pack);
|
||||||
|
}
|
||||||
|
$data .= $chunk;
|
||||||
|
}
|
||||||
|
$this->pos += $length;
|
||||||
|
return $data;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Repository;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Backend\IBackend;
|
||||||
|
|
||||||
|
/** The location as Repository sees it: closes the PackReader's open stream before every other request. */
|
||||||
|
final class PackReaderGuard implements IBackend {
|
||||||
|
public function __construct(
|
||||||
|
private IBackend $inner,
|
||||||
|
private PackReader $reader,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @param resource $stream */
|
||||||
|
public function put(string $path, $stream): void {
|
||||||
|
$this->reader->close();
|
||||||
|
$this->inner->put($path, $stream);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function get(string $path) {
|
||||||
|
$this->reader->close();
|
||||||
|
return $this->inner->get($path);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getMany(array $paths): \Generator {
|
||||||
|
$this->reader->close();
|
||||||
|
return $this->inner->getMany($paths);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getRange(string $path, int $offset, int $length): string {
|
||||||
|
$this->reader->close();
|
||||||
|
return $this->inner->getRange($path, $offset, $length);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function exists(string $path): bool {
|
||||||
|
$this->reader->close();
|
||||||
|
return $this->inner->exists($path);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function move(string $from, string $to): void {
|
||||||
|
$this->reader->close();
|
||||||
|
$this->inner->move($from, $to);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function list(string $prefix): array {
|
||||||
|
$this->reader->close();
|
||||||
|
return $this->inner->list($prefix);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function listFiles(string $dir): array {
|
||||||
|
$this->reader->close();
|
||||||
|
return $this->inner->listFiles($dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function delete(string $path): void {
|
||||||
|
$this->reader->close();
|
||||||
|
$this->inner->delete($path);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -32,6 +32,7 @@ final class PackWriter {
|
|||||||
private StreamCipher $cipher,
|
private StreamCipher $cipher,
|
||||||
private BlobIndex $index,
|
private BlobIndex $index,
|
||||||
private int $targetSize = 32 * 1048576,
|
private int $targetSize = 32 * 1048576,
|
||||||
|
private ?int $maxBytes = null,
|
||||||
) {
|
) {
|
||||||
$this->reset();
|
$this->reset();
|
||||||
}
|
}
|
||||||
@@ -49,11 +50,27 @@ final class PackWriter {
|
|||||||
return strlen($encrypted);
|
return strlen($encrypted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Add an already encrypted blob as-is (repacking: no decryption needed, the blob is bound to its id, not to a pack). */
|
||||||
|
public function addRaw(string $id, string $encrypted, int $rawLength, int $flags): void {
|
||||||
|
fwrite($this->buffer, $encrypted);
|
||||||
|
$this->entries[] = ['id' => $id, 'offset' => $this->size, 'length' => strlen($encrypted), 'raw' => $rawLength, 'flags' => $flags];
|
||||||
|
$this->size += strlen($encrypted);
|
||||||
|
if ($this->size >= $this->targetSize) {
|
||||||
|
$this->flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Upload the current pack (if any) and record it in the index. */
|
/** Upload the current pack (if any) and record it in the index. */
|
||||||
public function flush(): void {
|
public function flush(): void {
|
||||||
if ($this->entries === []) {
|
if ($this->entries === []) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if ($this->maxBytes !== null && $this->index->storedBytes() + $this->size > $this->maxBytes) {
|
||||||
|
$this->index->dropPending(array_column($this->entries, 'id'));
|
||||||
|
$this->reset();
|
||||||
|
throw new QuotaExceededException(sprintf('Storage limit of this location reached (%.1f GiB): backup stopped',
|
||||||
|
$this->maxBytes / 1073741824));
|
||||||
|
}
|
||||||
$packId = bin2hex(random_bytes(16));
|
$packId = bin2hex(random_bytes(16));
|
||||||
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
|
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
|
||||||
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Repository;
|
||||||
|
|
||||||
|
final class QuotaExceededException extends RepositoryException {
|
||||||
|
}
|
||||||
+591
-22
@@ -28,10 +28,17 @@ final class Repository {
|
|||||||
public const FORMAT = 1;
|
public const FORMAT = 1;
|
||||||
public const BLOB_SIZE = 4 * 1048576;
|
public const BLOB_SIZE = 4 * 1048576;
|
||||||
private const FLAG_DEFLATE = 1;
|
private const FLAG_DEFLATE = 1;
|
||||||
|
/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
|
||||||
|
private const VERIFY_READ_ATTEMPTS = 3;
|
||||||
|
|
||||||
private StreamCipher $cipher;
|
private StreamCipher $cipher;
|
||||||
private BlobIndex $index;
|
private BlobIndex $index;
|
||||||
private ?PackWriter $runPacks = null;
|
private ?PackWriter $runPacks = null;
|
||||||
|
private ?int $maxBytes = null;
|
||||||
|
private Catalog $catalog;
|
||||||
|
private PackReader $packReader;
|
||||||
|
/** verify() lists the pack folders from this many packs on (see presentPacks()); tests lower it. */
|
||||||
|
public int $listPacksAbove = 256;
|
||||||
|
|
||||||
private function __construct(
|
private function __construct(
|
||||||
private IBackend $backend,
|
private IBackend $backend,
|
||||||
@@ -39,7 +46,9 @@ final class Repository {
|
|||||||
private string $repositoryId,
|
private string $repositoryId,
|
||||||
) {
|
) {
|
||||||
$this->cipher = new StreamCipher($keys);
|
$this->cipher = new StreamCipher($keys);
|
||||||
$this->index = new BlobIndex($backend, $this->cipher);
|
$this->packReader = new PackReader($backend);
|
||||||
|
$this->backend = new PackReaderGuard($backend, $this->packReader);
|
||||||
|
$this->index = new BlobIndex($this->backend, $this->cipher);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||||
@@ -52,7 +61,7 @@ final class Repository {
|
|||||||
* copy (KeyRing::wrap) stored in the repository config, so the repository can be opened
|
* copy (KeyRing::wrap) stored in the repository config, so the repository can be opened
|
||||||
* with the passphrase from the recovery kit even when the server is gone.
|
* with the passphrase from the recovery kit even when the server is gone.
|
||||||
*/
|
*/
|
||||||
public static function initWithKey(IBackend $backend, KeyRing $keys, array $wrappedKey): self {
|
public static function initWithKey(IBackend $backend, KeyRing $keys, array $wrappedKey, ?ICatalogAnchor $anchor = null): self {
|
||||||
if ($backend->exists('config')) {
|
if ($backend->exists('config')) {
|
||||||
throw new RepositoryException('A repository already exists at this location');
|
throw new RepositoryException('A repository already exists at this location');
|
||||||
}
|
}
|
||||||
@@ -60,11 +69,13 @@ final class Repository {
|
|||||||
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
|
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
|
||||||
'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))];
|
'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))];
|
||||||
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
||||||
return new self($backend, $keys, $id);
|
$repo = new self($backend, $keys, $id);
|
||||||
|
$repo->loadCatalog($anchor);
|
||||||
|
return $repo;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
|
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
|
||||||
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null): self {
|
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null): self {
|
||||||
$config = self::readConfig($backend);
|
$config = self::readConfig($backend);
|
||||||
try {
|
try {
|
||||||
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
|
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
|
||||||
@@ -79,9 +90,32 @@ final class Repository {
|
|||||||
$repo->index->setCache($cache, $config['id']);
|
$repo->index->setCache($cache, $config['id']);
|
||||||
}
|
}
|
||||||
$repo->index->load();
|
$repo->index->load();
|
||||||
|
$repo->loadCatalog($anchor);
|
||||||
return $repo;
|
return $repo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function loadCatalog(?ICatalogAnchor $anchor): void {
|
||||||
|
$this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor);
|
||||||
|
$this->catalog->load(
|
||||||
|
fn () => array_map('basename', $this->backend->listFiles('snapshots')),
|
||||||
|
fn () => array_map('basename', $this->backend->listFiles('trash/info')),
|
||||||
|
fn () => $this->backend->list('users'));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Refuse uploads that would make the stored data larger than this (null = no limit). */
|
||||||
|
public function setMaxBytes(?int $maxBytes): void {
|
||||||
|
$this->maxBytes = $maxBytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function storedBytes(): int {
|
||||||
|
return $this->index->storedBytes();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Current catalog generation (for status and tests). */
|
||||||
|
public function catalogGeneration(): int {
|
||||||
|
return $this->catalog->gen;
|
||||||
|
}
|
||||||
|
|
||||||
/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
|
/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
|
||||||
public function indexDownloads(): int {
|
public function indexDownloads(): int {
|
||||||
return $this->index->downloaded;
|
return $this->index->downloaded;
|
||||||
@@ -101,14 +135,22 @@ final class Repository {
|
|||||||
return $config;
|
return $config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Replace the passphrase-wrapped key slots in this repository's config (after a slot change). */
|
||||||
|
public function updateWrappedKey(array $wrappedKey): void {
|
||||||
|
$config = self::readConfig($this->backend);
|
||||||
|
$config['key'] = $wrappedKey;
|
||||||
|
self::putString($this->backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
||||||
|
}
|
||||||
|
|
||||||
public function id(): string {
|
public function id(): string {
|
||||||
return $this->repositoryId;
|
return $this->repositoryId;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||||
$config = self::readConfig($backend);
|
$config = self::readConfig($backend);
|
||||||
$repo = new self($backend, KeyRing::unwrap($config['key'], $passphrase), $config['id']);
|
$repo = new self($backend, KeyRing::unwrapAny($config['key'], $passphrase), $config['id']);
|
||||||
$repo->index->load();
|
$repo->index->load();
|
||||||
|
$repo->loadCatalog(null); // disaster recovery on a new server: no anchor yet, trust the newest valid chain
|
||||||
return $repo;
|
return $repo;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,7 +163,7 @@ final class Repository {
|
|||||||
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
||||||
$source = rtrim($source, '/');
|
$source = rtrim($source, '/');
|
||||||
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
||||||
$packs = new PackWriter($this->backend, $this->cipher, $this->index);
|
$packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
||||||
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
||||||
|
|
||||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
@@ -149,35 +191,65 @@ final class Repository {
|
|||||||
rewind($encrypted);
|
rewind($encrypted);
|
||||||
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
||||||
fclose($tree);
|
fclose($tree);
|
||||||
|
$this->catalog->snapshots[$snapshotId] = true;
|
||||||
|
$this->catalog->write();
|
||||||
|
|
||||||
return ['snapshot' => $snapshotId] + $stats;
|
return ['snapshot' => $snapshotId] + $stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Restore a snapshot (or only paths starting with $prefix) into a local directory. */
|
/**
|
||||||
public function restore(string $snapshotId, string $target, string $prefix = ''): int {
|
* Restore a snapshot (or only paths starting with $prefix) into a local directory.
|
||||||
|
*
|
||||||
|
* @param bool $stripPrefix write "$target/<path without $prefix>" instead of
|
||||||
|
* "$target/<full logical path>" (disaster recovery: restoring e.g. "data/" or
|
||||||
|
* "config/" directly into the real data/config directory, not a subdirectory
|
||||||
|
* named after the prefix). Ignored when $prefix is '' (nothing to strip).
|
||||||
|
*/
|
||||||
|
public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null): int {
|
||||||
|
if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) {
|
||||||
|
throw new RepositoryException('Invalid path prefix');
|
||||||
|
}
|
||||||
$count = 0;
|
$count = 0;
|
||||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
$dest = rtrim($target, '/') . '/' . $entry['p'];
|
$relPath = ($stripPrefix && $prefix !== '') ? substr($entry['p'], strlen(rtrim($prefix, '/')) + 1) : $entry['p'];
|
||||||
|
$dest = rtrim($target, '/') . '/' . $relPath;
|
||||||
if (!is_dir(dirname($dest))) {
|
if (!is_dir(dirname($dest))) {
|
||||||
mkdir(dirname($dest), 0700, true);
|
mkdir(dirname($dest), 0700, true);
|
||||||
}
|
}
|
||||||
|
// Defence in depth: the resolved directory must stay below the target (no symlink escapes).
|
||||||
|
$base = realpath($target);
|
||||||
|
$dir = realpath(dirname($dest));
|
||||||
|
if ($base === false || $dir === false || ($dir !== $base && !str_starts_with($dir, $base . '/')) || is_link($dest)) {
|
||||||
|
throw new RepositoryException('Refusing to write outside the restore directory: ' . $entry['p']);
|
||||||
|
}
|
||||||
// Write next to the destination and rename after all blobs verified: at most one file
|
// Write next to the destination and rename after all blobs verified: at most one file
|
||||||
// is ever present twice.
|
// is ever present twice.
|
||||||
$tmp = $dest . '.ngb-restore';
|
$tmp = $dest . '.ngb-restore-' . bin2hex(random_bytes(6));
|
||||||
$out = fopen($tmp, 'wb');
|
$out = @fopen($tmp, 'xb'); // fails if anything (including a symlink) already exists there
|
||||||
|
if ($out === false) {
|
||||||
|
throw new RepositoryException('Cannot create a temporary file for ' . $entry['p']);
|
||||||
|
}
|
||||||
foreach ($entry['b'] as $id) {
|
foreach ($entry['b'] as $id) {
|
||||||
fwrite($out, $this->loadBlob($id));
|
fwrite($out, $this->loadBlob($id));
|
||||||
|
if ($heartbeat !== null) {
|
||||||
|
$heartbeat();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
fclose($out);
|
fclose($out);
|
||||||
if (filesize($tmp) !== $entry['s']) {
|
if (filesize($tmp) !== $entry['s']) {
|
||||||
unlink($tmp);
|
unlink($tmp);
|
||||||
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
||||||
}
|
}
|
||||||
rename($tmp, $dest);
|
if (!rename($tmp, $dest)) {
|
||||||
touch($dest, $entry['m']);
|
@unlink($tmp);
|
||||||
|
throw new RepositoryException('Cannot put the restored file in place: ' . $entry['p']);
|
||||||
|
}
|
||||||
|
if (!touch($dest, $entry['m'])) {
|
||||||
|
throw new RepositoryException('Cannot set the modification time of ' . $entry['p']);
|
||||||
|
}
|
||||||
$count++;
|
$count++;
|
||||||
}
|
}
|
||||||
return $count;
|
return $count;
|
||||||
@@ -188,13 +260,22 @@ final class Repository {
|
|||||||
* run is done, so the last pack is uploaded before anything references it.
|
* run is done, so the last pack is uploaded before anything references it.
|
||||||
*/
|
*/
|
||||||
public function blobWriter(array &$stats): BlobWriter {
|
public function blobWriter(array &$stats): BlobWriter {
|
||||||
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
||||||
$packs = $this->runPacks;
|
$packs = $this->runPacks;
|
||||||
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
|
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
|
||||||
return $this->storeBlob($data, $packs, $stats);
|
return $this->storeBlob($data, $packs, $stats);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** See BlobIndex::holdCache(): no database writes until releaseIndexCache(). */
|
||||||
|
public function holdIndexCache(): void {
|
||||||
|
$this->index->holdCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function releaseIndexCache(): void {
|
||||||
|
$this->index->releaseCache();
|
||||||
|
}
|
||||||
|
|
||||||
public function flushPacks(): void {
|
public function flushPacks(): void {
|
||||||
$this->runPacks?->flush();
|
$this->runPacks?->flush();
|
||||||
}
|
}
|
||||||
@@ -220,7 +301,7 @@ final class Repository {
|
|||||||
|
|
||||||
/** Store one blob of file data in the current run (deduplicated); returns its id. */
|
/** Store one blob of file data in the current run (deduplicated); returns its id. */
|
||||||
public function storeData(string $data, array &$stats): string {
|
public function storeData(string $data, array &$stats): string {
|
||||||
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
||||||
return $this->storeBlob($data, $this->runPacks, $stats);
|
return $this->storeBlob($data, $this->runPacks, $stats);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -231,6 +312,17 @@ final class Repository {
|
|||||||
* @param iterable<string> $lines
|
* @param iterable<string> $lines
|
||||||
*/
|
*/
|
||||||
public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
||||||
|
$this->putSnapshot($snapshotId, $meta, $lines);
|
||||||
|
$this->recordSnapshot($snapshotId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Store a snapshot object without touching the catalog. Use recordSnapshot() afterwards, from a
|
||||||
|
* repository opened inside the catalog lock, so the catalog write builds on the latest generation.
|
||||||
|
*
|
||||||
|
* @param iterable<string> $lines
|
||||||
|
*/
|
||||||
|
public function putSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
||||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n");
|
fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n");
|
||||||
foreach ($lines as $line) {
|
foreach ($lines as $line) {
|
||||||
@@ -246,6 +338,11 @@ final class Repository {
|
|||||||
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function recordSnapshot(string $snapshotId): void {
|
||||||
|
$this->catalog->snapshots[$snapshotId] = true;
|
||||||
|
$this->catalog->write();
|
||||||
|
}
|
||||||
|
|
||||||
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
||||||
public function snapshotEntries(string $snapshotId): array {
|
public function snapshotEntries(string $snapshotId): array {
|
||||||
return $this->loadEntries($snapshotId);
|
return $this->loadEntries($snapshotId);
|
||||||
@@ -261,6 +358,19 @@ final class Repository {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Relative path of non-empty segments, without ".", "..", NUL, backslash or control characters. */
|
||||||
|
public static function isSafePath(string $path): bool {
|
||||||
|
if ($path === '' || str_starts_with($path, '/') || preg_match('/[\x00-\x1f\\\\]/', $path)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
foreach (explode('/', $path) as $segment) {
|
||||||
|
if ($segment === '' || $segment === '.' || $segment === '..') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/** Small encrypted objects (manifests). */
|
/** Small encrypted objects (manifests). */
|
||||||
public function putObject(string $path, string $data): void {
|
public function putObject(string $path, string $data): void {
|
||||||
self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path));
|
self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path));
|
||||||
@@ -273,9 +383,280 @@ final class Repository {
|
|||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<string> */
|
/**
|
||||||
|
* Move a snapshot (and its database dump) to the trash on the location. Its data stays in
|
||||||
|
* use until purgeTrash() removes it after the deletion delay, so a malicious or mistaken
|
||||||
|
* cleanup can be undone with untrash() within that time.
|
||||||
|
*/
|
||||||
|
public function forget(string $snapshotId, string $by = ''): void {
|
||||||
|
$meta = $this->snapshotMeta($snapshotId);
|
||||||
|
$dbInTrash = null;
|
||||||
|
if (isset($meta['db']) && is_string($meta['db']) && $this->backend->exists($meta['db'])) {
|
||||||
|
// Its encryption is bound to its path (object:db/...), which changes on a move, so it
|
||||||
|
// must be decrypted and re-encrypted under the new path, not moved as raw bytes.
|
||||||
|
$dbInTrash = 'trash/' . $meta['db'];
|
||||||
|
$this->putObject($dbInTrash, $this->getObject($meta['db']));
|
||||||
|
}
|
||||||
|
// Copy first, switch the catalog, delete the originals last: a crash at any point leaves
|
||||||
|
// either the old state or the new one plus unreferenced leftovers, never a catalog entry
|
||||||
|
// pointing at a missing object. The snapshot object is bound to its id (not its path),
|
||||||
|
// so it can be copied as raw bytes.
|
||||||
|
$this->copyRaw('snapshots/' . $snapshotId, 'trash/snapshots/' . $snapshotId);
|
||||||
|
$this->putObject('trash/info/' . $snapshotId, json_encode(['forgottenAt' => time(), 'by' => $by, 'db' => $meta['db'] ?? null,
|
||||||
|
'dbInTrash' => $dbInTrash, 'time' => $meta['time'] ?? null, 'label' => $meta['label'] ?? ''], JSON_THROW_ON_ERROR));
|
||||||
|
unset($this->catalog->snapshots[$snapshotId]);
|
||||||
|
$this->catalog->trash[$snapshotId] = true;
|
||||||
|
$this->catalog->write();
|
||||||
|
if ($dbInTrash !== null) {
|
||||||
|
$this->backend->delete($meta['db']);
|
||||||
|
}
|
||||||
|
$this->backend->delete('snapshots/' . $snapshotId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Undo forget(): move a snapshot back from the trash. */
|
||||||
|
public function untrash(string $snapshotId): void {
|
||||||
|
$info = $this->trashInfo($snapshotId);
|
||||||
|
$restoreDb = $info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash']);
|
||||||
|
if ($restoreDb) {
|
||||||
|
$this->putObject($info['db'], $this->getObject($info['dbInTrash']));
|
||||||
|
}
|
||||||
|
// Same order as forget(): copy, switch the catalog, then delete the trash copies.
|
||||||
|
$this->copyRaw('trash/snapshots/' . $snapshotId, 'snapshots/' . $snapshotId);
|
||||||
|
$this->catalog->snapshots[$snapshotId] = true;
|
||||||
|
unset($this->catalog->trash[$snapshotId]);
|
||||||
|
$this->catalog->write();
|
||||||
|
if ($restoreDb) {
|
||||||
|
$this->backend->delete($info['dbInTrash']);
|
||||||
|
}
|
||||||
|
$this->backend->delete('trash/snapshots/' . $snapshotId);
|
||||||
|
$this->backend->delete('trash/info/' . $snapshotId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Copy an object as raw bytes, buffered locally (remote read streams cannot be uploaded directly everywhere). */
|
||||||
|
private function copyRaw(string $from, string $to): void {
|
||||||
|
$in = $this->backend->get($from);
|
||||||
|
$buffer = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
|
if ($buffer === false) {
|
||||||
|
fclose($in);
|
||||||
|
throw new RepositoryException("Cannot buffer $from");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (stream_copy_to_stream($in, $buffer) === false) {
|
||||||
|
throw new RepositoryException("Cannot read $from");
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
fclose($in);
|
||||||
|
}
|
||||||
|
rewind($buffer);
|
||||||
|
$this->backend->put($to, $buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Record a user_migration export's manifest in the catalog (rollback-protected like a snapshot). */
|
||||||
|
public function recordUserExport(string $manifestPath): void {
|
||||||
|
$this->catalog->userExports[$manifestPath] = true;
|
||||||
|
$this->catalog->write();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* User export manifests according to the verified catalog, newest first. A manifest the
|
||||||
|
* catalog lists but the location no longer has means it was deleted or hidden behind NG
|
||||||
|
* Backup's back, same as a missing snapshot (see listSnapshots()).
|
||||||
|
*
|
||||||
|
* @return list<string>
|
||||||
|
*/
|
||||||
|
public function listUserExports(?string $uid = null): array {
|
||||||
|
$prefix = $uid !== null ? 'users/' . $uid . '/' : 'users/';
|
||||||
|
$out = [];
|
||||||
|
foreach (array_keys($this->catalog->userExports) as $path) {
|
||||||
|
if (!str_starts_with($path, $prefix)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!$this->backend->exists($path)) {
|
||||||
|
throw new RepositoryException("User export $path is missing on the location (deleted or hidden outside NG Backup)");
|
||||||
|
}
|
||||||
|
$out[] = $path;
|
||||||
|
}
|
||||||
|
sort($out, SORT_STRING);
|
||||||
|
return array_reverse($out);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether $manifestPath is a user export the verified catalog actually recorded. */
|
||||||
|
public function isRecordedUserExport(string $manifestPath): bool {
|
||||||
|
return isset($this->catalog->userExports[$manifestPath]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Permanently delete a user export. Its blobs are freed by the next prune() like any other. */
|
||||||
|
public function forgetUserExport(string $manifestPath): void {
|
||||||
|
if (!isset($this->catalog->userExports[$manifestPath])) {
|
||||||
|
throw new RepositoryException("$manifestPath is not a recorded user export");
|
||||||
|
}
|
||||||
|
unset($this->catalog->userExports[$manifestPath]);
|
||||||
|
$this->catalog->write();
|
||||||
|
$this->backend->delete($manifestPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return array<string, array{forgottenAt:int, by:string, db:?string, dbInTrash:?string, time:?string, label:string}> */
|
||||||
|
public function trash(): array {
|
||||||
|
$out = [];
|
||||||
|
foreach (array_keys($this->catalog->trash) as $id) {
|
||||||
|
try {
|
||||||
|
$out[$id] = $this->trashInfo($id);
|
||||||
|
} catch (\Throwable) {
|
||||||
|
throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Permanently delete trash entries older than $delaySeconds. @return list<string> purged snapshot ids */
|
||||||
|
public function purgeTrash(int $delaySeconds): array {
|
||||||
|
$doomed = [];
|
||||||
|
foreach ($this->trash() as $id => $info) {
|
||||||
|
if (time() - $info['forgottenAt'] < $delaySeconds) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$doomed[$id] = $info;
|
||||||
|
unset($this->catalog->trash[$id]);
|
||||||
|
}
|
||||||
|
if ($doomed === []) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
$this->catalog->write();
|
||||||
|
foreach ($doomed as $id => $info) {
|
||||||
|
if ($info['dbInTrash'] !== null) {
|
||||||
|
$this->backend->delete($info['dbInTrash']);
|
||||||
|
}
|
||||||
|
$this->backend->delete('trash/snapshots/' . $id);
|
||||||
|
$this->backend->delete('trash/info/' . $id);
|
||||||
|
}
|
||||||
|
return array_keys($doomed);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function trashInfo(string $snapshotId): array {
|
||||||
|
return json_decode($this->getObject('trash/info/' . $snapshotId), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete packs no snapshot or user export uses any more, and repack packs where more than
|
||||||
|
* $repackThreshold of the bytes are unused. Must not run while a backup to this repository
|
||||||
|
* is in progress (its blobs are not referenced by a snapshot yet); callers hold the lock.
|
||||||
|
*
|
||||||
|
* @return array{packs:int, deleted:int, repacked:int, freedBytes:int, keptBlobs:int, unusedBlobs:int}
|
||||||
|
*/
|
||||||
|
public function prune(bool $dryRun = false, float $repackThreshold = 0.5, array $ignoreSnapshots = []): array {
|
||||||
|
// Deleting is decided on the authenticated index files on the location, never on the local cache.
|
||||||
|
$this->index->load(true);
|
||||||
|
$used = [];
|
||||||
|
$ignore = array_flip($ignoreSnapshots); // e.g. snapshots a dry run would forget first
|
||||||
|
foreach ($this->listSnapshots() as $sid) {
|
||||||
|
if (isset($ignore[$sid])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
foreach ($this->streamEntries($sid) as $e) {
|
||||||
|
foreach ($e['b'] as $b) {
|
||||||
|
$used[$b] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$meta = $this->snapshotMeta($sid);
|
||||||
|
if (isset($meta['db']) && $this->backend->exists($meta['db'])) {
|
||||||
|
foreach (json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
||||||
|
foreach ($t['blobs'] as $b) {
|
||||||
|
$used[$b] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Snapshots in the trash keep their data until purgeTrash() removed them.
|
||||||
|
foreach ($this->trash() as $sid => $info) {
|
||||||
|
foreach ($this->streamEntries($sid, 'trash/snapshots/') as $e) {
|
||||||
|
foreach ($e['b'] as $b) {
|
||||||
|
$used[$b] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash'])) {
|
||||||
|
foreach (json_decode($this->getObject($info['dbInTrash']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
||||||
|
foreach ($t['blobs'] as $b) {
|
||||||
|
$used[$b] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// listUserExports() throws when a recorded export's manifest is missing: then stop, since
|
||||||
|
// freeing its blobs would turn a tampering or storage alarm into irreversible deletion.
|
||||||
|
$exports = array_unique(array_merge($this->listUserExports(), $this->backend->list('users')));
|
||||||
|
foreach ($exports as $path) {
|
||||||
|
foreach (json_decode($this->getObject($path), true, 512, JSON_THROW_ON_ERROR)['entries'] as $e) {
|
||||||
|
foreach ($e['b'] ?? [] as $b) {
|
||||||
|
$used[$b] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$stats = ['packs' => 0, 'deleted' => 0, 'repacked' => 0, 'freedBytes' => 0, 'keptBlobs' => count($used), 'unusedBlobs' => 0];
|
||||||
|
$packs = $this->index->packs();
|
||||||
|
$stats['packs'] = count($packs);
|
||||||
|
$writer = null;
|
||||||
|
$toRemove = [];
|
||||||
|
foreach ($packs as $packId => $blobs) {
|
||||||
|
$total = 0;
|
||||||
|
$unused = 0;
|
||||||
|
foreach ($blobs as $b) {
|
||||||
|
$total += $b['length'];
|
||||||
|
if (!isset($used[$b['id']])) {
|
||||||
|
$unused += $b['length'];
|
||||||
|
$stats['unusedBlobs']++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($unused === 0 || ($unused < $total && $unused / $total <= $repackThreshold)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$stats['freedBytes'] += $unused;
|
||||||
|
if ($unused === $total) {
|
||||||
|
$stats['deleted']++;
|
||||||
|
} else {
|
||||||
|
$stats['repacked']++;
|
||||||
|
if (!$dryRun) {
|
||||||
|
// No size limit here: repacking is how space is freed.
|
||||||
|
$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
||||||
|
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
||||||
|
foreach ($blobs as $b) {
|
||||||
|
if (isset($used[$b['id']])) {
|
||||||
|
[, , , $raw, $flags] = $this->index->get($b['id']);
|
||||||
|
$writer->addRaw($b['id'], $this->backend->getRange($path, $b['offset'], $b['length']), $raw, $flags);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$toRemove[] = $packId;
|
||||||
|
}
|
||||||
|
if ($dryRun) {
|
||||||
|
return $stats;
|
||||||
|
}
|
||||||
|
// New packs and their index entries are durable before any old pack is removed.
|
||||||
|
$writer?->flush();
|
||||||
|
foreach ($toRemove as $packId) {
|
||||||
|
$this->index->removePack($packId);
|
||||||
|
$this->backend->delete('packs/' . substr($packId, 0, 2) . '/' . $packId);
|
||||||
|
}
|
||||||
|
return $stats;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Snapshots according to the verified catalog. A snapshot the catalog lists but the location
|
||||||
|
* no longer has means it was deleted or hidden behind NG Backup's back.
|
||||||
|
*
|
||||||
|
* @return list<string>
|
||||||
|
*/
|
||||||
public function listSnapshots(): array {
|
public function listSnapshots(): array {
|
||||||
return array_map('basename', $this->backend->list('snapshots'));
|
$present = array_flip(array_map('basename', $this->backend->listFiles('snapshots')));
|
||||||
|
$listed = array_keys($this->catalog->snapshots);
|
||||||
|
foreach ($listed as $id) {
|
||||||
|
if (!isset($present[$id])) {
|
||||||
|
throw new RepositoryException("Snapshot $id is missing on the location (deleted or hidden outside NG Backup)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort($listed);
|
||||||
|
return $listed;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function blobCount(): int {
|
public function blobCount(): int {
|
||||||
@@ -331,8 +712,13 @@ final class Repository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private function loadBlob(string $id): string {
|
private function loadBlob(string $id): string {
|
||||||
[$pack, $offset, $length, $raw, $flags] = $this->index->get($id);
|
[$pack, $offset, $length] = $this->index->get($id);
|
||||||
$encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
return $this->decodeBlob($id, $this->packReader->read($pack, $offset, $length));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */
|
||||||
|
private function decodeBlob(string $id, string $encrypted): string {
|
||||||
|
[, , , $raw, $flags] = $this->index->get($id);
|
||||||
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
||||||
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
||||||
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
||||||
@@ -355,6 +741,184 @@ final class Repository {
|
|||||||
return $this->streamEntries($snapshotId);
|
return $this->streamEntries($snapshotId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checksum audit of a snapshot: every blob it references (files and the database dump, when
|
||||||
|
* the snapshot has one) must be in the index, freshly re-downloaded and re-authenticated from
|
||||||
|
* the location rather than trusted from the local cache (same reasoning as prune()), and its
|
||||||
|
* pack must still be present. With $deep, each blob is also downloaded and decrypted, which
|
||||||
|
* verifies its AEAD authentication tag and its content hash against the blob id (loadBlob()
|
||||||
|
* already does both on every real read -- this just does it for everything in the snapshot,
|
||||||
|
* deliberately, rather than relying on it to come up during a later restore). A blob
|
||||||
|
* referenced by more than one file is only checked once; a missing database manifest is a
|
||||||
|
* failure, not something to silently skip over.
|
||||||
|
*
|
||||||
|
* @param ?callable(array{phase:string, done:int, total:?int, blobs?:int, bytes?:int}):void $heartbeat
|
||||||
|
* called often during the check, with how far it is: phase "index" (index files
|
||||||
|
* downloaded), "packs" (pack folders listed), "files" and "database" (files of the
|
||||||
|
* snapshot checked); callers use it to keep a time-boxed lease alive and to report
|
||||||
|
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
||||||
|
*/
|
||||||
|
public function verify(string $snapshotId, bool $deep = false, ?callable $heartbeat = null): array {
|
||||||
|
$heartbeat ??= static function (array $progress): void {
|
||||||
|
};
|
||||||
|
$meta = $this->snapshotMeta($snapshotId);
|
||||||
|
$filesTotal = isset($meta['stats']['files']) ? (int)$meta['stats']['files'] : null;
|
||||||
|
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
|
||||||
|
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
|
||||||
|
});
|
||||||
|
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
||||||
|
$seen = [];
|
||||||
|
[$packSeen, $listedDirs] = $this->presentPacks($heartbeat);
|
||||||
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||||
|
$stats['filesChecked']++;
|
||||||
|
foreach ($entry['b'] as $id) {
|
||||||
|
if (isset($seen[$id])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$seen[$id] = true;
|
||||||
|
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
|
||||||
|
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
|
||||||
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (isset($meta['db']) && is_string($meta['db'])) {
|
||||||
|
if (!$this->backend->exists($meta['db'])) {
|
||||||
|
$stats['missing'][] = $meta['db'];
|
||||||
|
} else {
|
||||||
|
$dbManifest = json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
foreach ($dbManifest['tables'] as $info) {
|
||||||
|
foreach ($info['blobs'] as $id) {
|
||||||
|
if (isset($seen[$id])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$seen[$id] = true;
|
||||||
|
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
|
||||||
|
$heartbeat(['phase' => 'database', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
|
||||||
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $stats;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which packs are on the location, from one listing per pack folder instead of a request per
|
||||||
|
* pack. A folder that cannot be listed, or comes back empty (which a storage also reports when
|
||||||
|
* it could not look), is left out: its packs are then checked one by one. With no more packs
|
||||||
|
* than folders nothing is listed: a request per pack is fewer requests then, and on FTP every
|
||||||
|
* request takes a data connection, of which a server may have only a few.
|
||||||
|
*
|
||||||
|
* @return array{0: array<string, bool>, 1: array<string, true>} pack id => present, and the
|
||||||
|
* folders that were listed
|
||||||
|
*/
|
||||||
|
private function presentPacks(callable $heartbeat): array {
|
||||||
|
$present = [];
|
||||||
|
$listed = [];
|
||||||
|
if ($this->index->packCount() <= $this->listPacksAbove) {
|
||||||
|
return [$present, $listed];
|
||||||
|
}
|
||||||
|
for ($i = 0; $i < 256; $i++) {
|
||||||
|
$dir = sprintf('%02x', $i);
|
||||||
|
try {
|
||||||
|
$files = $this->backend->listFiles('packs/' . $dir);
|
||||||
|
} catch (BackendException) {
|
||||||
|
$files = [];
|
||||||
|
}
|
||||||
|
foreach ($files as $path) {
|
||||||
|
$present[basename($path)] = true;
|
||||||
|
}
|
||||||
|
if ($files !== []) {
|
||||||
|
$listed[$dir] = true;
|
||||||
|
}
|
||||||
|
$heartbeat(['phase' => 'packs', 'done' => $i + 1, 'total' => 256]);
|
||||||
|
}
|
||||||
|
return [$present, $listed];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Same checks as verify(), for a user export: every blob its manifest references must be in
|
||||||
|
* the freshly re-authenticated index with its pack present; $deep also decrypts each.
|
||||||
|
*
|
||||||
|
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
||||||
|
*/
|
||||||
|
public function verifyUserExport(string $manifestPath, bool $deep = false, ?callable $heartbeat = null): array {
|
||||||
|
if (!$this->isRecordedUserExport($manifestPath)) {
|
||||||
|
throw new RepositoryException("$manifestPath is not a recorded user export for this location");
|
||||||
|
}
|
||||||
|
$heartbeat ??= static function (array $progress): void {
|
||||||
|
};
|
||||||
|
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
|
||||||
|
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
|
||||||
|
});
|
||||||
|
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
||||||
|
$seen = [];
|
||||||
|
$packSeen = [];
|
||||||
|
$manifest = json_decode($this->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
foreach ($manifest['entries'] as $entry) {
|
||||||
|
if (($entry['t'] ?? '') !== 'f') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$stats['filesChecked']++;
|
||||||
|
foreach ($entry['b'] ?? [] as $id) {
|
||||||
|
if (isset($seen[$id])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$seen[$id] = true;
|
||||||
|
$this->verifyBlob($id, $deep, $stats, $packSeen);
|
||||||
|
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => null,
|
||||||
|
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $stats;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param array<string, bool> $packSeen pack id => present; filled in as packs are checked
|
||||||
|
* @param array<string, true> $listedDirs pack folders whose listing is in $packSeen: a pack
|
||||||
|
* not in it is not there
|
||||||
|
*/
|
||||||
|
private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen, array $listedDirs = []): void {
|
||||||
|
if (!$this->index->has($id)) {
|
||||||
|
$stats['missing'][] = $id;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
[$pack] = $this->index->get($id);
|
||||||
|
$packExists = $packSeen[$pack] ??= isset($listedDirs[substr($pack, 0, 2)])
|
||||||
|
? false
|
||||||
|
: $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
|
||||||
|
if (!$packExists) {
|
||||||
|
$stats['missing'][] = $id;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$stats['blobsChecked']++;
|
||||||
|
if (!$deep) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Reading and checking are kept apart: a read that fails is usually a network or
|
||||||
|
// storage error, so it is retried and reported as unreadable (a pack that stays
|
||||||
|
// unreadable may still be truncated). Only a blob that was read but does not
|
||||||
|
// decrypt or match its id counts as failed: that is proof of damage.
|
||||||
|
[, $offset, $length] = $this->index->get($id);
|
||||||
|
$encrypted = null;
|
||||||
|
for ($attempt = 1; $attempt <= self::VERIFY_READ_ATTEMPTS && $encrypted === null; $attempt++) {
|
||||||
|
try {
|
||||||
|
$encrypted = $this->packReader->read($pack, $offset, $length);
|
||||||
|
} catch (\Exception $e) { // storage adapters throw their own exceptions; \Error (a bug) is not retried
|
||||||
|
$stats['readError'] = $e->getMessage();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($encrypted === null) {
|
||||||
|
$stats['unreadable'][] = $id;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$stats['bytesChecked'] += strlen($this->decodeBlob($id, $encrypted));
|
||||||
|
} catch (\Exception) { // damage; an \Error is a bug and must not be reported as damage
|
||||||
|
$stats['failed'][] = $id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** The meta record (first line) of a snapshot. */
|
/** The meta record (first line) of a snapshot. */
|
||||||
public function snapshotMeta(string $snapshotId): array {
|
public function snapshotMeta(string $snapshotId): array {
|
||||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
@@ -368,15 +932,20 @@ final class Repository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
||||||
private function streamEntries(string $snapshotId): \Generator {
|
private function streamEntries(string $snapshotId, string $dir = 'snapshots/'): \Generator {
|
||||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
$fh = $this->backend->get('snapshots/' . $snapshotId);
|
$fh = $this->backend->get($dir . $snapshotId);
|
||||||
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
||||||
fclose($fh);
|
fclose($fh);
|
||||||
rewind($plain);
|
rewind($plain);
|
||||||
fgets($plain); // meta line
|
fgets($plain); // meta line
|
||||||
while (($line = fgets($plain)) !== false) {
|
while (($line = fgets($plain)) !== false) {
|
||||||
yield json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
$entry = json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
if (!is_array($entry) || !is_string($entry['p'] ?? null) || !self::isSafePath($entry['p'])
|
||||||
|
|| !is_int($entry['s'] ?? null) || !is_array($entry['b'] ?? null)) {
|
||||||
|
throw new RepositoryException('Invalid entry in snapshot ' . $snapshotId);
|
||||||
|
}
|
||||||
|
yield $entry;
|
||||||
}
|
}
|
||||||
fclose($plain);
|
fclose($plain);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
use OCA\NgBackup\AppInfo\Application;
|
||||||
|
use OCA\NgBackup\Db\Run;
|
||||||
|
use OCA\NgBackup\Db\RunMapper;
|
||||||
|
use OCA\NgBackup\Db\Target;
|
||||||
|
use OCP\IGroupManager;
|
||||||
|
use OCP\Notification\IManager;
|
||||||
|
|
||||||
|
/** Notifies administrators about failed runs and locations without a recent successful backup. */
|
||||||
|
final class AlertService {
|
||||||
|
public const STALE_AFTER = 2 * 86400;
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private IManager $notifications,
|
||||||
|
private IGroupManager $groups,
|
||||||
|
private RunMapper $runs,
|
||||||
|
private \OCP\IAppConfig $appConfig,
|
||||||
|
private \OCP\EventDispatcher\IEventDispatcher $events,
|
||||||
|
private \OCP\IUserSession $session,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Security-relevant change (recovery kit downloaded, location removed, append-only off,
|
||||||
|
* retention changed, snapshots forgotten): written to the audit log (admin_audit) and sent
|
||||||
|
* to every administrator, so a single compromised admin account cannot do it unnoticed.
|
||||||
|
*
|
||||||
|
* @param array<string, string> $params
|
||||||
|
*/
|
||||||
|
public function actor(): string {
|
||||||
|
return $this->session->getUser()?->getUID() ?? (PHP_SAPI === 'cli' ? 'occ' : 'unknown');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function securityEvent(string $event, array $params = [], bool $notify = true): void {
|
||||||
|
$actor = $this->actor();
|
||||||
|
$params = ['actor' => $actor] + $params;
|
||||||
|
$this->events->dispatchTyped(new \OCP\Log\Audit\CriticalActionPerformedEvent(
|
||||||
|
'NG Backup: %s by %s ' . json_encode(array_diff_key($params, ['actor' => 1]), JSON_UNESCAPED_SLASHES), ['event' => $event, 'actor' => $actor]));
|
||||||
|
if ($notify) {
|
||||||
|
$this->notifyAdmins('security_event', $event . '-' . time(), ['event' => $event] + $params);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function runFailed(Run $run, string $targetName): void {
|
||||||
|
$this->notifyAdmins('run_failed', 'run-' . $run->getId(),
|
||||||
|
['kind' => $run->getKind(), 'target' => $targetName, 'error' => mb_strimwidth((string)$run->getError(), 0, 300, '…')]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A backup that finished but is missing unreadable files, or holds files that changed while read. */
|
||||||
|
public function runWarnings(Run $run, string $targetName, int $unreadable, int $changing, array $examples): void {
|
||||||
|
$this->notifyAdmins('run_warnings', 'run-' . $run->getId(), ['target' => $targetName,
|
||||||
|
'unreadable' => (string)$unreadable, 'changing' => (string)$changing,
|
||||||
|
'examples' => mb_strimwidth(implode(', ', array_merge($examples['unreadable'], $examples['changing'])), 0, 300, '…')]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remember how much is stored on a location (for the settings page) and warn administrators
|
||||||
|
* once a day when it is at 80% of its limit or more.
|
||||||
|
*/
|
||||||
|
public function storedBytes(Target $target, int $bytes): void {
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, 'stored_' . $target->getId(), (string)$bytes);
|
||||||
|
$limit = $target->getMaxBytes();
|
||||||
|
if ($limit === null || $limit <= 0 || $bytes < 0.8 * $limit) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$key = 'quota_notified_' . $target->getId();
|
||||||
|
if ($this->appConfig->getValueString(Application::APP_ID, $key, '') === date('Y-m-d')) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, $key, date('Y-m-d'));
|
||||||
|
$this->notifyAdmins('quota_warning', 'quota-' . $target->getId() . '-' . date('Y-m-d'), [
|
||||||
|
'target' => $target->getName(), 'percent' => (string)round($bytes / $limit * 100),
|
||||||
|
'used' => self::gib($bytes), 'limit' => self::gib($limit)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function gib(int $bytes): string {
|
||||||
|
$g = $bytes / 1073741824;
|
||||||
|
return sprintf($g < 10 ? '%.2f GiB' : '%.1f GiB', $g);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function lastStoredBytes(Target $target): ?int {
|
||||||
|
$v = $this->appConfig->getValueString(Application::APP_ID, 'stored_' . $target->getId(), '');
|
||||||
|
return $v === '' ? null : (int)$v;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** At most one notification per location per day. */
|
||||||
|
public function checkStale(Target $target): void {
|
||||||
|
$last = $this->runs->lastFinished($target->getId(), BackupService::KIND_FULL);
|
||||||
|
$lastTime = $last?->getFinishedAt() ?? $target->getCreatedAt();
|
||||||
|
if (time() - $lastTime < self::STALE_AFTER || $this->runs->findRunning($target->getId(), BackupService::KIND_FULL) !== []) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$key = 'stale_notified_' . $target->getId();
|
||||||
|
if ($this->appConfig->getValueString(Application::APP_ID, $key, '') === date('Y-m-d')) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, $key, date('Y-m-d'));
|
||||||
|
$this->notifyAdmins('backup_stale', 'stale-' . $target->getId() . '-' . date('Y-m-d'),
|
||||||
|
['target' => $target->getName(), 'since' => $last ? date('Y-m-d H:i', $lastTime) : '']);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function notifyAdmins(string $subject, string $objectId, array $params): void {
|
||||||
|
foreach ($this->groups->get('admin')?->getUsers() ?? [] as $admin) {
|
||||||
|
$n = $this->notifications->createNotification();
|
||||||
|
$n->setApp(Application::APP_ID)
|
||||||
|
->setUser($admin->getUID())
|
||||||
|
->setDateTime(new \DateTime())
|
||||||
|
->setObject($subject, $objectId)
|
||||||
|
->setSubject($subject, $params);
|
||||||
|
$this->notifications->notify($n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+179
-17
@@ -14,8 +14,11 @@ use OCA\NgBackup\Db\Snapshot;
|
|||||||
use OCA\NgBackup\Db\SnapshotMapper;
|
use OCA\NgBackup\Db\SnapshotMapper;
|
||||||
use OCA\NgBackup\Db\Target;
|
use OCA\NgBackup\Db\Target;
|
||||||
use OCA\NgBackup\Job\BackupRun;
|
use OCA\NgBackup\Job\BackupRun;
|
||||||
|
use OCA\NgBackup\Repository\RepositoryException;
|
||||||
|
use OCP\AppFramework\Db\DoesNotExistException;
|
||||||
use OCP\App\IAppManager;
|
use OCP\App\IAppManager;
|
||||||
use OCP\IConfig;
|
use OCP\IConfig;
|
||||||
|
use OCP\DB\QueryBuilder\IQueryBuilder;
|
||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
use OCP\Lock\ILockingProvider;
|
use OCP\Lock\ILockingProvider;
|
||||||
use OCP\Lock\LockedException;
|
use OCP\Lock\LockedException;
|
||||||
@@ -28,7 +31,7 @@ use Psr\Log\LoggerInterface;
|
|||||||
* Excluded by default: previews, log files, the updater's backups and each user's cache and
|
* Excluded by default: previews, log files, the updater's backups and each user's cache and
|
||||||
* temporary uploads (all regenerable or temporary).
|
* temporary uploads (all regenerable or temporary).
|
||||||
*/
|
*/
|
||||||
class BackupService {
|
final class BackupService {
|
||||||
public const KIND_FULL = 'full';
|
public const KIND_FULL = 'full';
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
@@ -41,17 +44,74 @@ class BackupService {
|
|||||||
private IAppManager $appManager,
|
private IAppManager $appManager,
|
||||||
private ILockingProvider $locking,
|
private ILockingProvider $locking,
|
||||||
private LoggerInterface $logger,
|
private LoggerInterface $logger,
|
||||||
|
private AlertService $alerts,
|
||||||
|
private LeaseService $leases,
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
public function start(Target $target, string $label = ''): Run {
|
public function start(Target $target, string $label = ''): Run {
|
||||||
if ($this->keys->recoveryKitConfirmation() === null) {
|
if (!$this->keys->everConfirmed()) {
|
||||||
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
|
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
|
||||||
}
|
}
|
||||||
if ($this->config->getSystemValue('objectstore', null) !== null) {
|
if ($this->config->getSystemValue('objectstore', null) !== null) {
|
||||||
throw new \RuntimeException('Object storage as primary storage is not supported yet');
|
throw new \RuntimeException('Object storage as primary storage is not supported yet');
|
||||||
}
|
}
|
||||||
$running = $this->runs->findRunning($target->getId());
|
// Check and create the run under the location's exclusive lock, so a cleanup cannot start
|
||||||
|
// in between (it takes the same lock and refuses while a run exists).
|
||||||
|
// A run that is already in progress is simply continued; no lease needed for that.
|
||||||
|
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
||||||
|
if ($running !== []) {
|
||||||
|
return $running[0];
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return $this->leases->with(PruneService::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $this->createRun($target, $label));
|
||||||
|
} catch (LockedException) {
|
||||||
|
throw new \RuntimeException('A cleanup or another start for ' . $target->getName() . ' is running; try again later');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The expected size of the files phase, for the progress shown while it runs: the size of the
|
||||||
|
* previous snapshot, or for a first backup the users' files, versions and trash as Nextcloud's
|
||||||
|
* file cache knows them (approximate: other data in the data directory is not counted).
|
||||||
|
*
|
||||||
|
* @return array{bytes: int, approximate: bool}
|
||||||
|
*/
|
||||||
|
private function estimateBytes(Run $run, array $files): array {
|
||||||
|
if (($files['parent'] ?? null) !== null) {
|
||||||
|
try {
|
||||||
|
return ['bytes' => $this->snapshots->findOne($run->getTargetId(), (string)$files['parent'])->getBytes() ?? 0, 'approximate' => false];
|
||||||
|
} catch (\Throwable) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->select('numeric_id')->from('storages')
|
||||||
|
->where($qb->expr()->like('id', $qb->createNamedParameter('home::%')))
|
||||||
|
->orWhere($qb->expr()->like('id', $qb->createNamedParameter('object::user:%')));
|
||||||
|
$result = $qb->executeQuery();
|
||||||
|
$storages = [];
|
||||||
|
while (($id = $result->fetchOne()) !== false) {
|
||||||
|
$storages[] = (int)$id;
|
||||||
|
}
|
||||||
|
$result->closeCursor();
|
||||||
|
$bytes = 0;
|
||||||
|
foreach (array_chunk($storages, 500) as $chunk) {
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->select($qb->func()->sum('size'))->from('filecache')
|
||||||
|
->where($qb->expr()->in('storage', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
|
||||||
|
->andWhere($qb->expr()->in('path_hash', $qb->createNamedParameter(array_map('md5', ['files', 'files_versions', 'files_trashbin']), IQueryBuilder::PARAM_STR_ARRAY)))
|
||||||
|
->andWhere($qb->expr()->gt('size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
|
||||||
|
$bytes += (int)$qb->executeQuery()->fetchOne();
|
||||||
|
}
|
||||||
|
return ['bytes' => $bytes, 'approximate' => true];
|
||||||
|
} catch (\Throwable) {
|
||||||
|
return ['bytes' => 0, 'approximate' => true];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function createRun(Target $target, string $label): Run {
|
||||||
|
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
||||||
if ($running !== []) {
|
if ($running !== []) {
|
||||||
return $running[0];
|
return $running[0];
|
||||||
}
|
}
|
||||||
@@ -86,34 +146,85 @@ class BackupService {
|
|||||||
* run locked and returns immediately.
|
* run locked and returns immediately.
|
||||||
*/
|
*/
|
||||||
public function step(Run $run, float $deadline): Run {
|
public function step(Run $run, float $deadline): Run {
|
||||||
$lockKey = 'ng_backup/run/' . $run->getId();
|
// Leases expire on their own: a step killed by a time limit or OOM blocks nothing for long.
|
||||||
|
// A step can run well past its deadline (the commit of a large snapshot writes its tree and
|
||||||
|
// index), so the lease holds at least 15 minutes; should it still expire and another process
|
||||||
|
// take the run over, the checks below keep the two from overwriting each other.
|
||||||
|
$ttl = (int)min(3600, max(900, $deadline - microtime(true) + 600));
|
||||||
try {
|
try {
|
||||||
$this->locking->acquireLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
|
$runLease = $this->leases->acquire('run/' . $run->getId(), LeaseService::EXCLUSIVE, $ttl);
|
||||||
} catch (LockedException) {
|
} catch (LockedException) {
|
||||||
|
return $run; // another process is doing this step
|
||||||
|
}
|
||||||
|
// Shared lease on the location while this step writes: a cleanup (exclusive) cannot run now.
|
||||||
|
try {
|
||||||
|
$targetLease = $this->leases->acquire('ng_backup/target/' . $run->getTargetId(), LeaseService::SHARED, $ttl);
|
||||||
|
} catch (LockedException) {
|
||||||
|
$this->leases->release($runLease);
|
||||||
return $run;
|
return $run;
|
||||||
}
|
}
|
||||||
|
$loadedAt = null;
|
||||||
try {
|
try {
|
||||||
$run = $this->runs->find($run->getId()); // fresh state after acquiring the lock
|
$run = $this->runs->find($run->getId()); // fresh state after acquiring the lock
|
||||||
if ($run->getStatus() !== Run::RUNNING) {
|
if ($run->getStatus() !== Run::RUNNING || $run->getKind() !== self::KIND_FULL) {
|
||||||
return $run;
|
return $run;
|
||||||
}
|
}
|
||||||
|
$loadedAt = $run->getUpdatedAt();
|
||||||
$target = $this->targets->get((string)$run->getTargetId());
|
$target = $this->targets->get((string)$run->getTargetId());
|
||||||
$repo = $this->targets->repository($target);
|
$repo = $this->targets->repository($target);
|
||||||
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
|
||||||
if ($run->getPhase() === 'db') {
|
if ($run->getPhase() === 'db') {
|
||||||
@set_time_limit(0); // the dump is one transaction; it cannot be split across requests yet
|
$dumper = new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_'));
|
||||||
$dbStats = [];
|
$dbStats = $state['dbStats'] ?? [];
|
||||||
$manifest = (new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_')))->dump($repo, $dbStats);
|
// With system cron the whole dump usually fits in one step (one transaction, fully
|
||||||
|
// consistent). Under web cron it continues per step; the snapshot then says so.
|
||||||
|
$dump = $dumper->dumpStep($repo, $state['dump'] ?? $dumper->startDump(), $deadline, $dbStats);
|
||||||
|
$state['dbStats'] = $dbStats;
|
||||||
|
if ($dump['done']) {
|
||||||
|
$manifest = $dumper->manifest($dump);
|
||||||
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
|
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
|
||||||
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
|
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
|
||||||
$state['dbStats'] = $dbStats + ['tables' => count($manifest['tables']),
|
$state['files']['meta']['dbConsistent'] = $manifest['consistent'];
|
||||||
'rows' => array_sum(array_column($manifest['tables'], 'rows'))];
|
$state['dbStats'] += ['tables' => count($manifest['tables']), 'rows' => array_sum(array_column($manifest['tables'], 'rows')),
|
||||||
|
'steps' => $dump['steps'], 'consistent' => $manifest['consistent']];
|
||||||
|
unset($state['dump']);
|
||||||
$run->setPhase('files');
|
$run->setPhase('files');
|
||||||
} else {
|
} else {
|
||||||
|
$state['dump'] = $dump;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$state['files']['estimate'] ??= $this->estimateBytes($run, $state['files']);
|
||||||
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
|
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
|
||||||
|
if ($state['files']['phase'] === 'commit') {
|
||||||
|
// Catalog write under the catalog lock, from a repository opened inside it: a per-user
|
||||||
|
// export or restore may have written the catalog since this step opened $repo.
|
||||||
|
try {
|
||||||
|
$this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300,
|
||||||
|
fn () => $this->targets->repository($target)->recordSnapshot((string)$state['files']['snapshot']));
|
||||||
|
$state['files']['phase'] = 'done';
|
||||||
|
} catch (LockedException|RepositoryException) {
|
||||||
|
// Another catalog write is in progress (or just raced us to the next
|
||||||
|
// generation): the next step retries this commit on the fresh catalog.
|
||||||
|
}
|
||||||
|
}
|
||||||
if ($state['files']['phase'] === 'done') {
|
if ($state['files']['phase'] === 'done') {
|
||||||
$this->recordSnapshot($run, $state['files']);
|
$this->recordSnapshot($run, $state['files']);
|
||||||
|
// The snapshot is committed: what follows only reports on it and must not turn
|
||||||
|
// the run into a failure.
|
||||||
|
try {
|
||||||
|
$this->alerts->storedBytes($target, $repo->storedBytes());
|
||||||
|
$stats = $state['files']['stats'];
|
||||||
|
if (($stats['unreadable'] ?? 0) > 0 || ($stats['changing'] ?? 0) > 0) {
|
||||||
|
$warnings = BackupRun::warnings($state['files']);
|
||||||
|
$this->logger->warning(sprintf('NG Backup run %d to %s: %d unreadable (missing from the snapshot): %s; %d changed while being read: %s',
|
||||||
|
$run->getId(), $target->getName(), $stats['unreadable'] ?? 0, implode(', ', $warnings['unreadable']),
|
||||||
|
$stats['changing'] ?? 0, implode(', ', $warnings['changing'])));
|
||||||
|
$this->alerts->runWarnings($run, $target->getName(), (int)($stats['unreadable'] ?? 0), (int)($stats['changing'] ?? 0), $warnings);
|
||||||
|
}
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$this->logger->warning('NG Backup run ' . $run->getId() . ': reporting on the finished backup failed: ' . $e->getMessage(), ['exception' => $e]);
|
||||||
|
}
|
||||||
$run->setStatus(Run::DONE);
|
$run->setStatus(Run::DONE);
|
||||||
$run->setPhase('done');
|
$run->setPhase('done');
|
||||||
$run->setFinishedAt(time());
|
$run->setFinishedAt(time());
|
||||||
@@ -121,24 +232,49 @@ class BackupService {
|
|||||||
}
|
}
|
||||||
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
|
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
|
||||||
$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
|
$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
|
||||||
$run->setUpdatedAt(time());
|
// Another process may have taken the run over after this step's lease expired: then its
|
||||||
return $this->runs->update($run);
|
// writes stand and this step's are dropped.
|
||||||
|
return $this->runs->updateIfUnchanged($run, $loadedAt) ?? $this->runs->find($run->getId());
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
if ($loadedAt === null) {
|
||||||
|
// The run could not even be loaded: there is nothing fresh to mark as failed, and
|
||||||
|
// the caller's copy may be stale.
|
||||||
|
$this->logger->error('NG Backup run ' . $run->getId() . ': step could not load the run: ' . $e->getMessage(), ['exception' => $e]);
|
||||||
|
return $run;
|
||||||
|
}
|
||||||
$run->setStatus(Run::FAILED);
|
$run->setStatus(Run::FAILED);
|
||||||
$run->setError($e->getMessage());
|
$run->setError($e->getMessage());
|
||||||
$run->setFinishedAt(time());
|
$run->setFinishedAt(time());
|
||||||
$run->setUpdatedAt(time());
|
$failed = $this->runs->updateIfUnchanged($run, $loadedAt);
|
||||||
return $this->runs->update($run);
|
if ($failed === null) {
|
||||||
|
// Another process that took the run over may have finished it: its result stands.
|
||||||
|
$this->logger->warning('NG Backup run ' . $run->getId() . ': a step that another process had taken over ended with: ' . $e->getMessage());
|
||||||
|
return $this->runs->find($run->getId());
|
||||||
|
}
|
||||||
|
$run = $failed;
|
||||||
|
$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
||||||
|
try {
|
||||||
|
$this->alerts->runFailed($run, isset($target) ? $target->getName() : '#' . $run->getTargetId());
|
||||||
|
} catch (\Throwable) {
|
||||||
|
}
|
||||||
|
return $run;
|
||||||
} finally {
|
} finally {
|
||||||
$this->locking->releaseLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
|
$this->leases->release($targetLease);
|
||||||
|
$this->leases->release($runLease);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Run to completion in this process (occ, system cron), reporting progress. */
|
/** Run to completion in this process (occ, system cron), reporting progress. */
|
||||||
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
|
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
|
||||||
while ($run->getStatus() === Run::RUNNING) {
|
while ($run->getStatus() === Run::RUNNING) {
|
||||||
|
$before = $run->getUpdatedAt();
|
||||||
$run = $this->step($run, microtime(true) + $stepSeconds);
|
$run = $this->step($run, microtime(true) + $stepSeconds);
|
||||||
|
if ($run->getStatus() === Run::RUNNING && $run->getUpdatedAt() === $before) {
|
||||||
|
// Step was not possible (another process holds the run, or a crashed step's lease has
|
||||||
|
// not expired yet): wait instead of spinning.
|
||||||
|
sleep(5);
|
||||||
|
$run = $this->runs->find($run->getId());
|
||||||
|
}
|
||||||
if ($progress !== null) {
|
if ($progress !== null) {
|
||||||
$progress($run);
|
$progress($run);
|
||||||
}
|
}
|
||||||
@@ -154,6 +290,11 @@ class BackupService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private function recordSnapshot(Run $run, array $files): void {
|
private function recordSnapshot(Run $run, array $files): void {
|
||||||
|
try {
|
||||||
|
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
|
||||||
|
return; // already recorded, by an earlier attempt at this step
|
||||||
|
} catch (DoesNotExistException) {
|
||||||
|
}
|
||||||
$s = new Snapshot();
|
$s = new Snapshot();
|
||||||
$s->setTargetId($run->getTargetId());
|
$s->setTargetId($run->getTargetId());
|
||||||
$s->setSnapshotId($files['snapshot']);
|
$s->setSnapshotId($files['snapshot']);
|
||||||
@@ -162,7 +303,19 @@ class BackupService {
|
|||||||
$s->setCreatedAt(time());
|
$s->setCreatedAt(time());
|
||||||
$s->setFiles($files['stats']['files']);
|
$s->setFiles($files['stats']['files']);
|
||||||
$s->setBytes($files['stats']['bytes']);
|
$s->setBytes($files['stats']['bytes']);
|
||||||
|
try {
|
||||||
$this->snapshots->insert($s);
|
$this->snapshots->insert($s);
|
||||||
|
} catch (\OCP\DB\Exception $e) {
|
||||||
|
// Recorded by another process between the check above and this insert?
|
||||||
|
if ($e->getReason() !== \OCP\DB\Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
|
||||||
|
} catch (DoesNotExistException) {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<string> */
|
/** @return list<string> */
|
||||||
@@ -177,6 +330,15 @@ class BackupService {
|
|||||||
'data/*/cache',
|
'data/*/cache',
|
||||||
'data/*/uploads',
|
'data/*/uploads',
|
||||||
];
|
];
|
||||||
|
// SQLite: the database file is in the data directory; it is backed up by the logical dump,
|
||||||
|
// and copying the live file would give an inconsistent copy.
|
||||||
|
if ($this->config->getSystemValueString('dbtype') === 'sqlite3') {
|
||||||
|
$dbName = $this->config->getSystemValueString('dbname', 'owncloud');
|
||||||
|
$exclude[] = "data/$dbName.db";
|
||||||
|
$exclude[] = "data/$dbName.db-journal";
|
||||||
|
$exclude[] = "data/$dbName.db-wal";
|
||||||
|
$exclude[] = "data/$dbName.db-shm";
|
||||||
|
}
|
||||||
$logFile = $this->config->getSystemValueString('logfile', '');
|
$logFile = $this->config->getSystemValueString('logfile', '');
|
||||||
if ($logFile !== '' && str_starts_with($logFile, $dataDir . '/')) {
|
if ($logFile !== '' && str_starts_with($logFile, $dataDir . '/')) {
|
||||||
$exclude[] = 'data/' . substr($logFile, strlen($dataDir) + 1);
|
$exclude[] = 'data/' . substr($logFile, strlen($dataDir) + 1);
|
||||||
|
|||||||
+241
-11
@@ -22,11 +22,15 @@ use OCP\Security\ICrypto;
|
|||||||
* into the recovery kit, so backups can be restored on a fresh server with the passphrase.
|
* into the recovery kit, so backups can be restored on a fresh server with the passphrase.
|
||||||
* - The admin must confirm having stored the recovery kit before the first backup runs.
|
* - The admin must confirm having stored the recovery kit before the first backup runs.
|
||||||
*/
|
*/
|
||||||
class KeyService {
|
final class KeyService {
|
||||||
private const KEY_MASTER = 'master_key';
|
private const KEY_MASTER = 'master_key';
|
||||||
private const KEY_WRAPPED = 'master_key_wrapped';
|
private const KEY_WRAPPED = 'master_key_wrapped';
|
||||||
private const KEY_CONFIRMED = 'recovery_kit_confirmed';
|
private const KEY_CONFIRMED = 'recovery_kit_confirmed';
|
||||||
private const KEY_FINGERPRINT = 'master_key_fingerprint';
|
private const KEY_FINGERPRINT = 'master_key_fingerprint';
|
||||||
|
private const KEY_DELETE_DELAY = 'delete_delay_days';
|
||||||
|
private const KEY_KIT_VERSION = 'recovery_kit_version';
|
||||||
|
private const KEY_KIT_CODE = 'recovery_kit_code';
|
||||||
|
public const DEFAULT_DELETE_DELAY = 7;
|
||||||
|
|
||||||
private ?KeyRing $cached = null;
|
private ?KeyRing $cached = null;
|
||||||
|
|
||||||
@@ -40,7 +44,15 @@ class KeyService {
|
|||||||
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true) !== '';
|
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true) !== '';
|
||||||
}
|
}
|
||||||
|
|
||||||
public function initialize(#[\SensitiveParameter] string $passphrase): void {
|
/**
|
||||||
|
* @param int $deleteDelayDays how long forgotten snapshots stay in the trash on the location
|
||||||
|
* before their data is really deleted. Fixed here, at installation:
|
||||||
|
* there is deliberately no way to change it later from NG Backup.
|
||||||
|
*/
|
||||||
|
public function initialize(#[\SensitiveParameter] string $passphrase, int $deleteDelayDays = self::DEFAULT_DELETE_DELAY, string $label = 'Slot 1'): void {
|
||||||
|
if ($deleteDelayDays < 1 || $deleteDelayDays > 365) {
|
||||||
|
throw new \InvalidArgumentException('The deletion delay must be between 1 and 365 days');
|
||||||
|
}
|
||||||
if ($this->isInitialized()) {
|
if ($this->isInitialized()) {
|
||||||
throw new \RuntimeException('A backup key already exists');
|
throw new \RuntimeException('A backup key already exists');
|
||||||
}
|
}
|
||||||
@@ -50,13 +62,95 @@ class KeyService {
|
|||||||
$keys = KeyRing::generate();
|
$keys = KeyRing::generate();
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
||||||
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED,
|
$this->cached = $keys;
|
||||||
json_encode($keys->wrap($passphrase), JSON_THROW_ON_ERROR), true, true);
|
$this->saveSlots([['slot' => 1, 'label' => self::cleanLabel($label), 'created' => time(), 'key' => $keys->wrap($passphrase)]]);
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
||||||
|
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, $deleteDelayDays, true);
|
||||||
|
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, 1, true);
|
||||||
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_CONFIRMED);
|
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_CONFIRMED);
|
||||||
$this->cached = $keys;
|
$this->cached = $keys;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Disaster recovery on a fresh installation: adopt the master key from a recovery kit
|
||||||
|
* (occ backup:key:kit on the original installation) instead of generating a new one.
|
||||||
|
* Afterwards occ backup:target:add opens the existing repository on the location with this
|
||||||
|
* key, instead of creating a new, empty one.
|
||||||
|
*
|
||||||
|
* @param array<string, mixed> $kit as produced by recoveryKit()
|
||||||
|
* @throws \OCA\NgBackup\Crypto\CryptoException wrong passphrase, or a damaged/foreign kit
|
||||||
|
*/
|
||||||
|
public function importRecoveryKit(array $kit, #[\SensitiveParameter] string $passphrase): void {
|
||||||
|
if ($this->isInitialized()) {
|
||||||
|
throw new \RuntimeException('A backup key already exists');
|
||||||
|
}
|
||||||
|
if (!isset($kit['wrapped_key'])) {
|
||||||
|
throw new \InvalidArgumentException('Not a NG Backup recovery kit file');
|
||||||
|
}
|
||||||
|
$keys = KeyRing::unwrapAny($kit['wrapped_key'], $passphrase);
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
||||||
|
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
||||||
|
$this->cached = $keys;
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, json_encode($kit['wrapped_key'], JSON_THROW_ON_ERROR), true, true);
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
||||||
|
// Kits from before this field existed fall back to the default.
|
||||||
|
$delay = (int)($kit['delete_delay_days'] ?? self::DEFAULT_DELETE_DELAY);
|
||||||
|
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, ($delay >= 1 && $delay <= 365) ? $delay : self::DEFAULT_DELETE_DELAY, true);
|
||||||
|
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, (int)($kit['kit_version'] ?? 1), true);
|
||||||
|
// Having and successfully using a real kit file is stronger proof than the normal
|
||||||
|
// download-and-confirm flow it replaces.
|
||||||
|
$this->confirmRecoveryKit('cli-import');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The master key and the recovery kit code are stored encrypted with Nextcloud's own
|
||||||
|
* instance secret (ICrypto), not with themselves; the wrapped key is not (it is sealed with
|
||||||
|
* the admin's passphrase instead), but Nextcloud's own AppConfig additionally encrypts any
|
||||||
|
* "sensitive" value with the instance secret regardless, underneath whatever we store. So
|
||||||
|
* changing that secret (occ backup:restore:full merging config.php) would otherwise leave
|
||||||
|
* all three permanently undecryptable. Call $applySecretChange (which must be the only thing
|
||||||
|
* that changes the secret) through here instead: all three are decrypted first, then
|
||||||
|
* re-encrypted under whatever secret is active once $applySecretChange returns.
|
||||||
|
*
|
||||||
|
* Not crash-safe: a process kill between the delete and the re-save of one of these keys
|
||||||
|
* would leave it missing rather than merely stale. Disaster recovery is run once, by hand,
|
||||||
|
* right after occ maintenance:install; a dedicated two-phase write is not justified yet for
|
||||||
|
* that narrow a window.
|
||||||
|
*/
|
||||||
|
public function reencryptForSecretRotation(callable $applySecretChange): mixed {
|
||||||
|
$master = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true);
|
||||||
|
$wrapped = $this->appConfig->getValueString(Application::APP_ID, self::KEY_WRAPPED, '', true);
|
||||||
|
$kitCodeRaw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_KIT_CODE, '', true);
|
||||||
|
$kitCode = $kitCodeRaw !== '' ? json_decode($kitCodeRaw, true, 512, JSON_THROW_ON_ERROR) : null;
|
||||||
|
$masterPlain = $master !== '' ? $this->crypto->decrypt($master) : null;
|
||||||
|
$kitCodePlain = $kitCode !== null && isset($kitCode['code']) ? $this->crypto->decrypt($kitCode['code']) : null;
|
||||||
|
|
||||||
|
$result = $applySecretChange();
|
||||||
|
$this->cached = null; // the master key is about to be re-saved; do not keep the pre-rotation copy
|
||||||
|
|
||||||
|
// Nextcloud's own AppConfig additionally encrypts a "sensitive" value itself (tied to
|
||||||
|
// the same secret, underneath our own encrypt() above); setValueString() on an existing
|
||||||
|
// key reads the old value first to compare, which would try to decrypt it with the new
|
||||||
|
// secret and fail the same way. Delete first so there is nothing to compare against.
|
||||||
|
if ($masterPlain !== null) {
|
||||||
|
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_MASTER);
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER, $this->crypto->encrypt($masterPlain), true, true);
|
||||||
|
}
|
||||||
|
if ($wrapped !== '') {
|
||||||
|
// Already plaintext JSON here: getValueString() above transparently undid
|
||||||
|
// AppConfig's own sensitive-value encryption; there is no encrypt() of our own to
|
||||||
|
// redo (wrapping is sealed with the passphrase, not the instance secret).
|
||||||
|
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_WRAPPED);
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, $wrapped, true, true);
|
||||||
|
}
|
||||||
|
if ($kitCodePlain !== null) {
|
||||||
|
$kitCode['code'] = $this->crypto->encrypt($kitCodePlain);
|
||||||
|
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_KIT_CODE);
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_KIT_CODE, json_encode($kitCode, JSON_THROW_ON_ERROR), true, true);
|
||||||
|
}
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
public function keyRing(): KeyRing {
|
public function keyRing(): KeyRing {
|
||||||
if ($this->cached === null) {
|
if ($this->cached === null) {
|
||||||
$enc = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true);
|
$enc = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true);
|
||||||
@@ -68,9 +162,99 @@ class KeyService {
|
|||||||
return $this->cached;
|
return $this->cached;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Deletion delay in days, fixed at installation (installations from before this setting get the default once). */
|
||||||
|
public function deleteDelayDays(): int {
|
||||||
|
$days = $this->appConfig->getValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, 0, true);
|
||||||
|
if ($days === 0 && $this->isInitialized()) {
|
||||||
|
$days = self::DEFAULT_DELETE_DELAY;
|
||||||
|
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, $days, true);
|
||||||
|
}
|
||||||
|
return $days;
|
||||||
|
}
|
||||||
|
|
||||||
|
public const MAX_SLOTS = 3;
|
||||||
|
|
||||||
|
/** @return list<array{slot:int, label:string, created:int}> passphrase slots, without key material */
|
||||||
|
public function slots(): array {
|
||||||
|
return array_map(fn ($s) => ['slot' => $s['slot'], 'label' => $s['label'], 'created' => $s['created']], $this->wrappedKey()['slots']);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Add a passphrase slot (e.g. one per person who must be able to restore). Returns the slot number. */
|
||||||
|
public function addSlot(#[\SensitiveParameter] string $passphrase, string $label): int {
|
||||||
|
self::checkPassphrase($passphrase);
|
||||||
|
$slots = $this->wrappedKey()['slots'];
|
||||||
|
if (count($slots) >= self::MAX_SLOTS) {
|
||||||
|
throw new \InvalidArgumentException('At most ' . self::MAX_SLOTS . ' passphrase slots');
|
||||||
|
}
|
||||||
|
$used = array_column($slots, 'slot');
|
||||||
|
$n = min(array_diff(range(1, self::MAX_SLOTS), $used));
|
||||||
|
$slots[] = ['slot' => $n, 'label' => self::cleanLabel($label), 'created' => time(), 'key' => $this->keyRing()->wrap($passphrase)];
|
||||||
|
usort($slots, fn ($a, $b) => $a['slot'] <=> $b['slot']);
|
||||||
|
$this->saveSlots($slots);
|
||||||
|
return $n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** New passphrase for a slot (e.g. its holder left). The master key and all backups stay the same. */
|
||||||
|
public function replaceSlot(int $slot, #[\SensitiveParameter] string $passphrase, ?string $label = null): void {
|
||||||
|
self::checkPassphrase($passphrase);
|
||||||
|
$slots = $this->wrappedKey()['slots'];
|
||||||
|
$found = false;
|
||||||
|
foreach ($slots as &$s) {
|
||||||
|
if ($s['slot'] === $slot) {
|
||||||
|
$s['key'] = $this->keyRing()->wrap($passphrase);
|
||||||
|
$s['created'] = time();
|
||||||
|
if ($label !== null) {
|
||||||
|
$s['label'] = self::cleanLabel($label);
|
||||||
|
}
|
||||||
|
$found = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unset($s);
|
||||||
|
if (!$found) {
|
||||||
|
throw new \InvalidArgumentException("There is no slot $slot");
|
||||||
|
}
|
||||||
|
$this->saveSlots($slots);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function removeSlot(int $slot): void {
|
||||||
|
$slots = array_values(array_filter($this->wrappedKey()['slots'], fn ($s) => $s['slot'] !== $slot));
|
||||||
|
if (count($slots) === count($this->wrappedKey()['slots'])) {
|
||||||
|
throw new \InvalidArgumentException("There is no slot $slot");
|
||||||
|
}
|
||||||
|
if ($slots === []) {
|
||||||
|
throw new \InvalidArgumentException('The last passphrase slot cannot be removed');
|
||||||
|
}
|
||||||
|
$this->saveSlots($slots);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Every change to the slots asks for a new recovery kit (and its confirmation). */
|
||||||
|
private function saveSlots(array $slots): void {
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, json_encode(['slots' => $slots], JSON_THROW_ON_ERROR), true, true);
|
||||||
|
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, $this->kitVersion() + 1, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function checkPassphrase(string $passphrase): void {
|
||||||
|
if (strlen($passphrase) < 12) {
|
||||||
|
throw new \InvalidArgumentException('The passphrase must be at least 12 characters');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function cleanLabel(string $label): string {
|
||||||
|
$label = trim(preg_replace('/[\x00-\x1f]/', '', $label) ?? '');
|
||||||
|
return mb_substr($label === '' ? 'Slot' : $label, 0, 64);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function kitVersion(): int {
|
||||||
|
return $this->appConfig->getValueInt(Application::APP_ID, self::KEY_KIT_VERSION, 1, true);
|
||||||
|
}
|
||||||
|
|
||||||
/** Passphrase-wrapped copy of the master key (for repository configs and the recovery kit). */
|
/** Passphrase-wrapped copy of the master key (for repository configs and the recovery kit). */
|
||||||
public function wrappedKey(): array {
|
public function wrappedKey(): array {
|
||||||
return json_decode($this->appConfig->getValueString(Application::APP_ID, self::KEY_WRAPPED, '{}', true), true, 512, JSON_THROW_ON_ERROR);
|
$data = json_decode($this->appConfig->getValueString(Application::APP_ID, self::KEY_WRAPPED, '{}', true), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
if (!isset($data['slots'])) {
|
||||||
|
$data = ['slots' => $data === [] ? [] : [['slot' => 1, 'label' => 'Slot 1', 'created' => 0, 'key' => $data]]];
|
||||||
|
}
|
||||||
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Short identifier of the key, shown in the UI and the kit (not secret). */
|
/** Short identifier of the key, shown in the UI and the kit (not secret). */
|
||||||
@@ -78,29 +262,75 @@ class KeyService {
|
|||||||
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_FINGERPRINT, '', true);
|
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_FINGERPRINT, '', true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return array{version:int, app:string, fingerprint:string, created:string, wrapped_key:array, instructions:string} */
|
/** @return array{version:int, app:string, fingerprint:string, kit_version:int, delete_delay_days:int, confirmation_code:string, created:string, wrapped_key:array, slots:list<array{slot:int, label:string, created:int}>, instructions:string} */
|
||||||
public function recoveryKit(): array {
|
public function recoveryKit(): array {
|
||||||
return [
|
return [
|
||||||
'version' => 1,
|
'version' => 1,
|
||||||
'app' => 'NG Backup',
|
'app' => 'NG Backup',
|
||||||
'fingerprint' => $this->fingerprint(),
|
'fingerprint' => $this->fingerprint(),
|
||||||
|
'kit_version' => $this->kitVersion(),
|
||||||
|
'delete_delay_days' => $this->deleteDelayDays(),
|
||||||
|
'confirmation_code' => $this->kitCode(),
|
||||||
'created' => gmdate('c'),
|
'created' => gmdate('c'),
|
||||||
'wrapped_key' => $this->wrappedKey(),
|
'wrapped_key' => $this->wrappedKey(),
|
||||||
'instructions' => 'Keep this file and your passphrase safe, outside this server. To restore on a new server: install NG Backup, '
|
'slots' => $this->slots(),
|
||||||
. 'add the backup location, and enter this kit with your passphrase. Without both, the backups cannot be decrypted.',
|
'instructions' => 'Keep this file safe, outside this server. Any one of the passphrases of the slots listed here opens it. '
|
||||||
|
. 'To restore on a new server: install NG Backup, add the backup location and enter this kit with one passphrase. '
|
||||||
|
. 'Without the kit (or the location) and a passphrase, the backups cannot be decrypted.',
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Short code printed in the current kit file. Typing it back when confirming proves the
|
||||||
|
* admin actually has the downloaded kit. A new code per kit version; the server only keeps
|
||||||
|
* its own copy encrypted with the instance secret.
|
||||||
|
*/
|
||||||
|
private function kitCode(): string {
|
||||||
|
$stored = json_decode($this->appConfig->getValueString(Application::APP_ID, self::KEY_KIT_CODE, '{}', true), true) ?: [];
|
||||||
|
if (($stored['version'] ?? 0) === $this->kitVersion() && isset($stored['code'])) {
|
||||||
|
return $this->crypto->decrypt($stored['code']);
|
||||||
|
}
|
||||||
|
$alphabet = 'ABCDEFGHJKLMNPQRSTUVWXYZ23456789'; // no 0/O, 1/I
|
||||||
|
$code = '';
|
||||||
|
for ($i = 0; $i < 8; $i++) {
|
||||||
|
$code .= $alphabet[random_int(0, strlen($alphabet) - 1)];
|
||||||
|
}
|
||||||
|
$code = substr($code, 0, 4) . '-' . substr($code, 4);
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_KIT_CODE,
|
||||||
|
json_encode(['version' => $this->kitVersion(), 'code' => $this->crypto->encrypt($code)], JSON_THROW_ON_ERROR), true, true);
|
||||||
|
return $code;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function checkKitCode(string $code): bool {
|
||||||
|
$stored = json_decode($this->appConfig->getValueString(Application::APP_ID, self::KEY_KIT_CODE, '{}', true), true) ?: [];
|
||||||
|
if (($stored['version'] ?? 0) !== $this->kitVersion() || !isset($stored['code'])) {
|
||||||
|
return false; // no kit downloaded for this version yet
|
||||||
|
}
|
||||||
|
$normalise = static fn (string $c) => strtoupper(preg_replace('/[^A-Za-z0-9]/', '', $c) ?? '');
|
||||||
|
return hash_equals($normalise($this->crypto->decrypt($stored['code'])), $normalise($code));
|
||||||
|
}
|
||||||
|
|
||||||
public function confirmRecoveryKit(string $uid): void {
|
public function confirmRecoveryKit(string $uid): void {
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_CONFIRMED,
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_CONFIRMED,
|
||||||
json_encode(['uid' => $uid, 'time' => time(), 'fingerprint' => $this->fingerprint()], JSON_THROW_ON_ERROR), true);
|
json_encode(['uid' => $uid, 'time' => time(), 'fingerprint' => $this->fingerprint(), 'kit_version' => $this->kitVersion()], JSON_THROW_ON_ERROR), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Confirmation for the current key, or null. */
|
/**
|
||||||
|
* Whether a recovery kit for this key was ever confirmed (any passphrase version). Backups
|
||||||
|
* keep running after a passphrase change, so changing it cannot be used to stop backups; the
|
||||||
|
* missing re-confirmation is reported as an error in the admin overview instead.
|
||||||
|
*/
|
||||||
|
public function everConfirmed(): bool {
|
||||||
|
$raw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_CONFIRMED, '', true);
|
||||||
|
$data = $raw === '' ? null : json_decode($raw, true);
|
||||||
|
return $data !== null && ($data['fingerprint'] ?? '') === $this->fingerprint();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Confirmation for the current key and kit version, or null. */
|
||||||
public function recoveryKitConfirmation(): ?array {
|
public function recoveryKitConfirmation(): ?array {
|
||||||
$raw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_CONFIRMED, '', true);
|
$raw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_CONFIRMED, '', true);
|
||||||
$data = $raw === '' ? null : json_decode($raw, true);
|
$data = $raw === '' ? null : json_decode($raw, true);
|
||||||
return ($data !== null && ($data['fingerprint'] ?? '') === $this->fingerprint()) ? $data : null;
|
return ($data !== null && ($data['fingerprint'] ?? '') === $this->fingerprint() && ($data['kit_version'] ?? 1) === $this->kitVersion()) ? $data : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function fingerprintOf(KeyRing $keys): string {
|
private function fingerprintOf(KeyRing $keys): string {
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Passphrase slots (like LUKS): up to three passphrases, each opening the same master key, so
|
||||||
|
* that one person leaving or dying does not lock the company out of its backups. Every change
|
||||||
|
* is written to all locations and announced to all administrators; a new recovery kit must be
|
||||||
|
* downloaded and confirmed afterwards (backups keep running meanwhile).
|
||||||
|
*/
|
||||||
|
final class KeySlotService {
|
||||||
|
public function __construct(
|
||||||
|
private KeyService $keys,
|
||||||
|
private TargetService $targets,
|
||||||
|
private AlertService $alerts,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public function add(#[\SensitiveParameter] string $passphrase, string $label): int {
|
||||||
|
$slot = $this->keys->addSlot($passphrase, $label);
|
||||||
|
$this->apply('slot_added', $slot);
|
||||||
|
return $slot;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function replace(int $slot, #[\SensitiveParameter] string $passphrase, ?string $label = null): void {
|
||||||
|
$this->keys->replaceSlot($slot, $passphrase, $label);
|
||||||
|
$this->apply('slot_replaced', $slot);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function remove(int $slot): void {
|
||||||
|
$this->keys->removeSlot($slot);
|
||||||
|
$this->apply('slot_removed', $slot);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function apply(string $event, int $slot): void {
|
||||||
|
$wrapped = $this->keys->wrappedKey();
|
||||||
|
$failed = [];
|
||||||
|
foreach ($this->targets->list() as $t) {
|
||||||
|
try {
|
||||||
|
$this->targets->repository($t)->updateWrappedKey($wrapped);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$failed[] = $t->getName();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$label = '';
|
||||||
|
foreach ($this->keys->slots() as $s) {
|
||||||
|
if ($s['slot'] === $slot) {
|
||||||
|
$label = $s['label'];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$this->alerts->securityEvent($event, ['slot' => (string)$slot, 'label' => $label]);
|
||||||
|
if ($failed !== []) {
|
||||||
|
throw new \RuntimeException('Saved, but these locations could not be updated (they still accept the previous passphrases): ' . implode(', ', $failed));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
use OCP\IDBConnection;
|
||||||
|
use OCP\Lock\ILockingProvider;
|
||||||
|
use OCP\Lock\LockedException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shared/exclusive leases with an expiry, stored in ngb_leases.
|
||||||
|
*
|
||||||
|
* Nextcloud's locks live up to an hour; a PHP process killed while holding one (web cron time
|
||||||
|
* limit, OOM, kill -9) would block a location that long. A lease expires on its own unless its
|
||||||
|
* holder keeps refreshing it. Nextcloud's lock is only held for the few milliseconds in which a
|
||||||
|
* lease is checked and written.
|
||||||
|
*/
|
||||||
|
final class LeaseService {
|
||||||
|
public const SHARED = 's';
|
||||||
|
public const EXCLUSIVE = 'x';
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private IDBConnection $db,
|
||||||
|
private ILockingProvider $locking,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return string lease id; throws LockedException when it conflicts with a live lease */
|
||||||
|
public function acquire(string $name, string $mode, int $ttl): string {
|
||||||
|
$guard = 'ng_backup/lease/' . $name;
|
||||||
|
$this->locking->acquireLock($guard, ILockingProvider::LOCK_EXCLUSIVE);
|
||||||
|
try {
|
||||||
|
$now = time();
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->delete('ngb_leases')
|
||||||
|
->where($qb->expr()->eq('name', $qb->createNamedParameter($name)))
|
||||||
|
->andWhere($qb->expr()->lt('expires_at', $qb->createNamedParameter($now)));
|
||||||
|
$qb->executeStatement();
|
||||||
|
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->select('mode')->from('ngb_leases')->where($qb->expr()->eq('name', $qb->createNamedParameter($name)));
|
||||||
|
$modes = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
|
||||||
|
if ($modes !== [] && ($mode === self::EXCLUSIVE || in_array(self::EXCLUSIVE, $modes, true))) {
|
||||||
|
throw new LockedException($name);
|
||||||
|
}
|
||||||
|
$id = bin2hex(random_bytes(16));
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->insert('ngb_leases')->values([
|
||||||
|
'id' => $qb->createNamedParameter($id),
|
||||||
|
'name' => $qb->createNamedParameter($name),
|
||||||
|
'mode' => $qb->createNamedParameter($mode),
|
||||||
|
'expires_at' => $qb->createNamedParameter($now + $ttl),
|
||||||
|
]);
|
||||||
|
$qb->executeStatement();
|
||||||
|
return $id;
|
||||||
|
} finally {
|
||||||
|
$this->locking->releaseLock($guard, ILockingProvider::LOCK_EXCLUSIVE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Extend a lease while its holder is still working. */
|
||||||
|
public function refresh(string $id, int $ttl): void {
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->update('ngb_leases')->set('expires_at', $qb->createNamedParameter(time() + $ttl))
|
||||||
|
->where($qb->expr()->eq('id', $qb->createNamedParameter($id)));
|
||||||
|
$qb->executeStatement();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function release(string $id): void {
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->delete('ngb_leases')->where($qb->expr()->eq('id', $qb->createNamedParameter($id)));
|
||||||
|
$qb->executeStatement();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Wraps $refresh so calling it often (per file, per row batch) only touches the database every $seconds. */
|
||||||
|
public static function throttled(callable $refresh, int $seconds = 60): \Closure {
|
||||||
|
$last = time();
|
||||||
|
return static function () use ($refresh, $seconds, &$last): void {
|
||||||
|
if (time() - $last >= $seconds) {
|
||||||
|
$refresh();
|
||||||
|
$last = time();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Run $fn under a lease; $refresh() can be called during long work. */
|
||||||
|
public function with(string $name, string $mode, int $ttl, callable $fn): mixed {
|
||||||
|
$id = $this->acquire($name, $mode, $ttl);
|
||||||
|
try {
|
||||||
|
return $fn(fn () => $this->refresh($id, $ttl));
|
||||||
|
} finally {
|
||||||
|
$this->release($id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,240 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
use OCA\NgBackup\AppInfo\Application;
|
||||||
|
use OCA\NgBackup\Db\RunMapper;
|
||||||
|
use OCA\NgBackup\Db\SnapshotMapper;
|
||||||
|
use OCA\NgBackup\Db\Target;
|
||||||
|
use OCP\IAppConfig;
|
||||||
|
use OCP\Lock\ILockingProvider;
|
||||||
|
use OCP\Lock\LockedException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Applies the retention policy to a location: forgets snapshots the policy does not keep and
|
||||||
|
* prunes data no remaining snapshot uses. Never runs on append-only locations, never while a
|
||||||
|
* backup to the same location is running (the location lock is shared with BackupService).
|
||||||
|
*/
|
||||||
|
final class PruneService {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private RunMapper $runs,
|
||||||
|
private SnapshotMapper $snapshots,
|
||||||
|
private IAppConfig $appConfig,
|
||||||
|
private ILockingProvider $locking,
|
||||||
|
private AlertService $alerts,
|
||||||
|
private KeyService $keys,
|
||||||
|
private LeaseService $leases,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public function untrash(Target $target, string $snapshotId): void {
|
||||||
|
$lease = $this->lockExclusive($target, 300);
|
||||||
|
try {
|
||||||
|
$this->doUntrash($target, $snapshotId);
|
||||||
|
} finally {
|
||||||
|
$this->leases->release($lease);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run $fn under the location's exclusive lock (to create a run atomically with respect to a
|
||||||
|
* cleanup, which takes the same lock and then refuses while a run exists).
|
||||||
|
*/
|
||||||
|
public function underStartLock(Target $target, callable $fn): mixed {
|
||||||
|
try {
|
||||||
|
return $this->leases->with(self::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $fn());
|
||||||
|
} catch (LockedException) {
|
||||||
|
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exclusive lock on the location, then make sure no run of any kind (backup, restore, export)
|
||||||
|
* is in progress: their data may not be referenced by a snapshot yet, or is being read.
|
||||||
|
*/
|
||||||
|
private function lockExclusive(Target $target, int $ttl): string {
|
||||||
|
try {
|
||||||
|
$lease = $this->leases->acquire(self::lockKey($target), LeaseService::EXCLUSIVE, $ttl);
|
||||||
|
} catch (LockedException) {
|
||||||
|
throw new \RuntimeException('A backup, restore or cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||||
|
}
|
||||||
|
if ($this->runs->findRunning($target->getId()) !== []) {
|
||||||
|
$this->leases->release($lease);
|
||||||
|
throw new \RuntimeException('A backup or restore of ' . $target->getName() . ' is running; try again later');
|
||||||
|
}
|
||||||
|
return $lease;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function doUntrash(Target $target, string $snapshotId): void {
|
||||||
|
$repo = $this->targets->repository($target);
|
||||||
|
if (!array_key_exists($snapshotId, $repo->trash())) {
|
||||||
|
throw new \InvalidArgumentException('That snapshot is not in the trash (or was already deleted)');
|
||||||
|
}
|
||||||
|
$repo->untrash($snapshotId);
|
||||||
|
// Put it back in the local snapshot list as well.
|
||||||
|
$meta = $repo->snapshotMeta($snapshotId);
|
||||||
|
$s = new \OCA\NgBackup\Db\Snapshot();
|
||||||
|
$s->setTargetId($target->getId());
|
||||||
|
$s->setSnapshotId($snapshotId);
|
||||||
|
$s->setKind((string)($meta['kind'] ?? 'full'));
|
||||||
|
$s->setLabel(($meta['label'] ?? '') !== '' ? $meta['label'] : null);
|
||||||
|
$s->setCreatedAt((int)strtotime((string)($meta['time'] ?? 'now')));
|
||||||
|
$s->setFiles((int)($meta['stats']['files'] ?? 0));
|
||||||
|
$s->setBytes((int)($meta['stats']['bytes'] ?? 0));
|
||||||
|
try {
|
||||||
|
$this->snapshots->insert($s);
|
||||||
|
} catch (\Throwable) {
|
||||||
|
}
|
||||||
|
$this->alerts->securityEvent('snapshot_untrashed', ['target' => $target->getName(), 'snapshot' => $snapshotId]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static function lockKey(Target $target): string {
|
||||||
|
return 'ng_backup/target/' . $target->getId();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A separate, short-lived exclusive lock serializing writes to the catalog itself. The
|
||||||
|
* per-target lock above only keeps readers/writers out while a cleanup runs; it does not
|
||||||
|
* stop two SHARED holders (e.g. two concurrent per-user backups, or one alongside a full
|
||||||
|
* backup) from both reading the same catalog generation and racing to write the next one,
|
||||||
|
* silently dropping whichever's change loses. The full-backup path avoids this today only
|
||||||
|
* because a Run row makes concurrent runs for the same target mutually exclusive; per-user
|
||||||
|
* backup/restore has no such row, so it takes this lock around the catalog write itself.
|
||||||
|
*/
|
||||||
|
public static function catalogLockKey(Target $target): string {
|
||||||
|
return self::lockKey($target) . '/catalog';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** How many of each user's own exports (occ backup:user:backup) to keep, newest first. */
|
||||||
|
public function userExportRetention(): int {
|
||||||
|
return $this->appConfig->getValueInt(Application::APP_ID, 'user_export_retention', 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setUserExportRetention(int $keep): void {
|
||||||
|
if ($keep < 1) {
|
||||||
|
throw new \InvalidArgumentException('Keep at least the last export');
|
||||||
|
}
|
||||||
|
$old = $this->userExportRetention();
|
||||||
|
$this->appConfig->setValueInt(Application::APP_ID, 'user_export_retention', $keep);
|
||||||
|
if ($old !== $keep) {
|
||||||
|
$this->alerts->securityEvent('user_export_retention_changed', ['keep' => (string)$keep]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Forgets a user's older exports once more than userExportRetention() exist for them,
|
||||||
|
* newest first; no trash, unlike a full snapshot (occ backup:user:backup is on-demand, not
|
||||||
|
* yet the primary safety net the deletion delay protects).
|
||||||
|
*
|
||||||
|
* @return array{kept: array<string, list<string>>, forgotten: list<string>}
|
||||||
|
*/
|
||||||
|
public function applyUserExports(Target $target, bool $dryRun = false): array {
|
||||||
|
if ($target->getAppendOnly()) {
|
||||||
|
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
|
||||||
|
}
|
||||||
|
$lease = $this->lockExclusive($target, 1800);
|
||||||
|
try {
|
||||||
|
$repo = $this->targets->repository($target);
|
||||||
|
$keep = $this->userExportRetention();
|
||||||
|
$byUser = [];
|
||||||
|
foreach ($repo->listUserExports() as $path) {
|
||||||
|
$uid = explode('/', $path)[1] ?? '';
|
||||||
|
$manifest = json_decode($repo->getObject($path), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
$byUser[$uid][$path] = (int)strtotime((string)($manifest['time'] ?? '@0'));
|
||||||
|
}
|
||||||
|
$kept = [];
|
||||||
|
$forget = [];
|
||||||
|
foreach ($byUser as $uid => $times) {
|
||||||
|
arsort($times);
|
||||||
|
$i = 0;
|
||||||
|
foreach (array_keys($times) as $path) {
|
||||||
|
if ($i++ < $keep) {
|
||||||
|
$kept[$uid][] = $path;
|
||||||
|
} else {
|
||||||
|
$forget[] = $path;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!$dryRun) {
|
||||||
|
foreach ($forget as $path) {
|
||||||
|
$repo->forgetUserExport($path);
|
||||||
|
}
|
||||||
|
if ($forget !== []) {
|
||||||
|
$this->alerts->securityEvent('user_exports_forgotten', ['target' => $target->getName(), 'count' => (string)count($forget)], false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ['kept' => $kept, 'forgotten' => $forget];
|
||||||
|
} finally {
|
||||||
|
$this->leases->release($lease);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function policy(): RetentionPolicy {
|
||||||
|
return RetentionPolicy::fromArray(json_decode($this->appConfig->getValueString(Application::APP_ID, 'retention', '{}'), true) ?: []);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setPolicy(RetentionPolicy $policy): void {
|
||||||
|
if ($policy->last < 1) {
|
||||||
|
throw new \InvalidArgumentException('Keep at least the last snapshot');
|
||||||
|
}
|
||||||
|
$old = $this->policy()->toArray();
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, 'retention', json_encode($policy->toArray(), JSON_THROW_ON_ERROR));
|
||||||
|
if ($old !== $policy->toArray()) {
|
||||||
|
$this->alerts->securityEvent('retention_changed', ['policy' => json_encode($policy->toArray())]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Snapshots the policy drops go to the trash on the location; trash entries older than the
|
||||||
|
* deletion delay are purged, and only then is their data freed.
|
||||||
|
*
|
||||||
|
* @return array{kept: array<string, list<string>>, forgotten: list<string>, purged: list<string>, prune: array}
|
||||||
|
*/
|
||||||
|
public function apply(Target $target, bool $dryRun = false, bool $scheduled = false): array {
|
||||||
|
if ($target->getAppendOnly()) {
|
||||||
|
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
|
||||||
|
}
|
||||||
|
$lease = $this->lockExclusive($target, 1800);
|
||||||
|
try {
|
||||||
|
$repo = $this->targets->repository($target);
|
||||||
|
$times = [];
|
||||||
|
foreach ($repo->listSnapshots() as $id) {
|
||||||
|
$meta = $repo->snapshotMeta($id);
|
||||||
|
$times[$id] = (int)strtotime((string)($meta['time'] ?? '@0'));
|
||||||
|
}
|
||||||
|
$keep = $this->policy()->keep($times);
|
||||||
|
$forget = array_values(array_diff(array_keys($times), array_keys($keep)));
|
||||||
|
$actor = $scheduled ? 'retention policy' : $this->alerts->actor();
|
||||||
|
$purged = [];
|
||||||
|
if (!$dryRun) {
|
||||||
|
foreach ($forget as $id) {
|
||||||
|
$repo->forget($id, $actor);
|
||||||
|
try {
|
||||||
|
$this->snapshots->delete($this->snapshots->findOne($target->getId(), $id));
|
||||||
|
} catch (\Throwable) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Also removes leftovers of interrupted runs. A dry run counts the snapshots it would forget as gone.
|
||||||
|
$delay = $this->keys->deleteDelayDays();
|
||||||
|
if (!$dryRun && $forget !== []) {
|
||||||
|
// Scheduled cleanup follows the policy: audit log only, no notification every week.
|
||||||
|
$this->alerts->securityEvent('snapshots_forgotten', ['target' => $target->getName(), 'count' => (string)count($forget),
|
||||||
|
'until' => date('Y-m-d', time() + $delay * 86400)], !$scheduled);
|
||||||
|
}
|
||||||
|
if (!$dryRun) {
|
||||||
|
$purged = $repo->purgeTrash($delay * 86400);
|
||||||
|
}
|
||||||
|
$prune = $repo->prune($dryRun);
|
||||||
|
return ['kept' => $keep, 'forgotten' => $forget, 'purged' => $purged, 'prune' => $prune];
|
||||||
|
} finally {
|
||||||
|
$this->leases->release($lease);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Db\DbDumper;
|
||||||
|
use OCA\NgBackup\Db\Target;
|
||||||
|
use OCP\App\IAppManager;
|
||||||
|
use OCP\IConfig;
|
||||||
|
use OCP\IDBConnection;
|
||||||
|
use OCP\Lock\LockedException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Disaster recovery on a fresh Nextcloud installation (same major version): replace the
|
||||||
|
* database, restore the data directory and merge the backed-up config.php, see PLAN.md 3.7.
|
||||||
|
*
|
||||||
|
* Deliberately NOT handled yet (phase 3): restoring in place over a live, already-used
|
||||||
|
* installation. The admin is expected to run this once, right after occ maintenance:install and
|
||||||
|
* occ backup:key:import/backup:target:add, before anyone else touches the instance.
|
||||||
|
*/
|
||||||
|
final class RestoreFullService {
|
||||||
|
/**
|
||||||
|
* Config keys that must keep pointing at THIS installation's actual, already-working
|
||||||
|
* database and data directory (set by occ maintenance:install), never at the old server's.
|
||||||
|
*/
|
||||||
|
private const CONNECTION_KEYS = [
|
||||||
|
'dbtype', 'dbhost', 'dbport', 'dbname', 'dbuser', 'dbpassword', 'dbpersistent',
|
||||||
|
'dbtableprefix', 'dbdriveroptions', 'datadirectory',
|
||||||
|
];
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private RestoreService $restore,
|
||||||
|
private LeaseService $leases,
|
||||||
|
private IConfig $config,
|
||||||
|
private IDBConnection $db,
|
||||||
|
private IAppManager $appManager,
|
||||||
|
private KeyService $keys,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether this installation can receive $snapshotId: same Nextcloud major version, and every
|
||||||
|
* app that was enabled at backup time already installed here (its tables must exist before
|
||||||
|
* the database restore can write into them).
|
||||||
|
*
|
||||||
|
* @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list<string>}
|
||||||
|
*/
|
||||||
|
public function checkCompatibility(Target $target, string $snapshotId): array {
|
||||||
|
$meta = $this->targets->repository($target)->snapshotMeta($snapshotId);
|
||||||
|
$currentMajor = explode('.', $this->config->getSystemValueString('version'))[0];
|
||||||
|
$snapshotMajor = explode('.', (string)($meta['nextcloud'] ?? ''))[0];
|
||||||
|
$enabled = array_flip($this->appManager->getEnabledApps());
|
||||||
|
$missingApps = array_values(array_diff(array_keys($meta['apps'] ?? []), array_keys($enabled)));
|
||||||
|
return [
|
||||||
|
'ok' => $currentMajor === $snapshotMajor && $missingApps === [],
|
||||||
|
'currentMajor' => $currentMajor,
|
||||||
|
'snapshotMajor' => $snapshotMajor,
|
||||||
|
'missingApps' => $missingApps,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool} */
|
||||||
|
public function restoreFull(Target $target, string $snapshotId): array {
|
||||||
|
$check = $this->checkCompatibility($target, $snapshotId);
|
||||||
|
if (!$check['ok']) {
|
||||||
|
throw new \RuntimeException($this->describeBlockers($check));
|
||||||
|
}
|
||||||
|
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId) {
|
||||||
|
$beat = LeaseService::throttled($refresh);
|
||||||
|
$repo = $this->targets->repository($target);
|
||||||
|
$meta = $repo->snapshotMeta($snapshotId);
|
||||||
|
|
||||||
|
$dbReport = null;
|
||||||
|
$manifest = [];
|
||||||
|
if (isset($meta['db'])) {
|
||||||
|
$manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
$prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_');
|
||||||
|
// ng_backup's own tables (and its own rows in the shared appconfig table) must
|
||||||
|
// survive this restore untouched: they describe the backup/key/target that is
|
||||||
|
// actively performing it right now, on THIS installation, not the one that was
|
||||||
|
// backed up. Overwriting them mid-restore would pull the rug from under it.
|
||||||
|
foreach (array_keys($manifest['tables']) as $table) {
|
||||||
|
/** @var string $table */
|
||||||
|
if (str_starts_with($table, $prefix . 'ngb_')) {
|
||||||
|
unset($manifest['tables'][$table]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Sequences are keyed separately from tables; excluding a table does not exclude
|
||||||
|
// its sequence, which would otherwise rewind a retained ngb_* table's sequence
|
||||||
|
// below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()).
|
||||||
|
foreach (array_keys($manifest['sequences'] ?? []) as $seq) {
|
||||||
|
/** @var string $seq */
|
||||||
|
if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) {
|
||||||
|
unset($manifest['sequences'][$seq]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$ownAppConfig = $this->preserveOwnAppConfig();
|
||||||
|
$dumper = new DbDumper($this->db->getInner(), $prefix);
|
||||||
|
try {
|
||||||
|
$dumper->restore($repo, $manifest, $dbReport, $beat);
|
||||||
|
} finally {
|
||||||
|
$this->restoreOwnAppConfig($ownAppConfig);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read (but do not yet apply) the backed-up config.php before touching the data
|
||||||
|
// directory: once applied, 'secret' may differ from what ng_backup's own Target
|
||||||
|
// options were encrypted with (see restoreFull()'s docblock), and every further
|
||||||
|
// repository access through $this->targets would fail to decrypt them.
|
||||||
|
/** @var array<string, mixed> $backupConfig */
|
||||||
|
$backupConfig = $this->fetchBackupConfig($target, $snapshotId);
|
||||||
|
|
||||||
|
$dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/');
|
||||||
|
$dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat);
|
||||||
|
|
||||||
|
// Last step: nothing after this may need $this->targets->repository($target) again.
|
||||||
|
if ($backupConfig !== []) {
|
||||||
|
$this->targets->reencryptOptionsAround(
|
||||||
|
fn (): mixed => $this->keys->reencryptForSecretRotation(fn (): mixed => $this->config->setSystemValues($backupConfig)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ['dbReport' => $dbReport ?? ['skipped' => [], 'restored' => []], 'dataFiles' => $dataFiles, 'configKeys' => count($backupConfig),
|
||||||
|
'dbConsistent' => (bool)($manifest['consistent'] ?? true)];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return array<string, mixed> config.php's $CONFIG from the backup, minus the connection keys */
|
||||||
|
private function fetchBackupConfig(Target $target, string $snapshotId): array {
|
||||||
|
$tmp = sys_get_temp_dir() . '/ngb-restore-config-' . bin2hex(random_bytes(6));
|
||||||
|
try {
|
||||||
|
$this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true);
|
||||||
|
$path = $tmp . '/config.php';
|
||||||
|
if (!is_file($path)) {
|
||||||
|
return []; // nothing to merge; the live config.php stays as occ maintenance:install made it
|
||||||
|
}
|
||||||
|
$backupConfig = (static function () use ($path): array {
|
||||||
|
$CONFIG = [];
|
||||||
|
include $path;
|
||||||
|
return $CONFIG;
|
||||||
|
})();
|
||||||
|
foreach (self::CONNECTION_KEYS as $key) {
|
||||||
|
unset($backupConfig[$key]);
|
||||||
|
}
|
||||||
|
return $backupConfig;
|
||||||
|
} finally {
|
||||||
|
$this->rmrf($tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return list<array<string, mixed>> raw appconfig rows (appid, configkey, configvalue, type, lazy) */
|
||||||
|
private function preserveOwnAppConfig(): array {
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->select('*')->from('appconfig')
|
||||||
|
->where($qb->expr()->eq('appid', $qb->createNamedParameter(\OCA\NgBackup\AppInfo\Application::APP_ID)));
|
||||||
|
/** @var list<array<string, mixed>> */
|
||||||
|
return $qb->executeQuery()->fetchAll();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param list<array<string, mixed>> $rows raw appconfig rows, as returned by preserveOwnAppConfig()
|
||||||
|
*
|
||||||
|
* Runs in its own transaction (DbDumper::restore() already committed its own by the time
|
||||||
|
* this is called): not a full guarantee against a crash spanning both, but means this step
|
||||||
|
* itself cannot leave ng_backup's appconfig rows half deleted, half reinserted.
|
||||||
|
*/
|
||||||
|
private function restoreOwnAppConfig(array $rows): void {
|
||||||
|
$this->db->beginTransaction();
|
||||||
|
try {
|
||||||
|
$qb = $this->db->getQueryBuilder();
|
||||||
|
$qb->delete('appconfig')
|
||||||
|
->where($qb->expr()->eq('appid', $qb->createNamedParameter(\OCA\NgBackup\AppInfo\Application::APP_ID)));
|
||||||
|
$qb->executeStatement();
|
||||||
|
foreach ($rows as $row) {
|
||||||
|
$ins = $this->db->getQueryBuilder();
|
||||||
|
$ins->insert('appconfig')->values([
|
||||||
|
'appid' => $ins->createNamedParameter($row['appid']),
|
||||||
|
'configkey' => $ins->createNamedParameter($row['configkey']),
|
||||||
|
'configvalue' => $ins->createNamedParameter($row['configvalue']),
|
||||||
|
'type' => $ins->createNamedParameter($row['type'], \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_INT),
|
||||||
|
'lazy' => $ins->createNamedParameter($row['lazy'], \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_INT),
|
||||||
|
]);
|
||||||
|
$ins->executeStatement();
|
||||||
|
}
|
||||||
|
$this->db->commit();
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$this->db->rollBack();
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function rmrf(string $dir): void {
|
||||||
|
if (!is_dir($dir)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
foreach (new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir, \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST) as $f) {
|
||||||
|
$f->isDir() ? rmdir($f->getPathname()) : unlink($f->getPathname());
|
||||||
|
}
|
||||||
|
rmdir($dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function describeBlockers(array $check): string {
|
||||||
|
$parts = [];
|
||||||
|
if ($check['currentMajor'] !== $check['snapshotMajor']) {
|
||||||
|
$parts[] = "this installation is Nextcloud {$check['currentMajor']}, the backup is from {$check['snapshotMajor']} (same major version required)";
|
||||||
|
}
|
||||||
|
if ($check['missingApps'] !== []) {
|
||||||
|
$parts[] = 'install these apps first (their tables must exist before the database restore can use them): '
|
||||||
|
. implode(', ', array_map(fn (string $a) => "occ app:install $a", $check['missingApps']));
|
||||||
|
}
|
||||||
|
return implode('; ', $parts);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function withSharedLock(Target $target, callable $fn): mixed {
|
||||||
|
try {
|
||||||
|
// Refreshed during the database and data restore (per row batch, per blob), so a
|
||||||
|
// cleanup cannot take over the location while a long restore is still reading it.
|
||||||
|
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 7200, $fn);
|
||||||
|
} catch (LockedException) {
|
||||||
|
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -28,7 +28,7 @@ use OCP\Files\NotFoundException;
|
|||||||
* - merge: into the original place, existing files are overwritten (old content kept as a version)
|
* - merge: into the original place, existing files are overwritten (old content kept as a version)
|
||||||
* - replace: like merge, and files that are not in the snapshot are moved to the trash
|
* - replace: like merge, and files that are not in the snapshot are moved to the trash
|
||||||
*/
|
*/
|
||||||
class RestoreService {
|
final class RestoreService {
|
||||||
public const MODE_NEW_FOLDER = 'new-folder';
|
public const MODE_NEW_FOLDER = 'new-folder';
|
||||||
public const MODE_MERGE = 'merge';
|
public const MODE_MERGE = 'merge';
|
||||||
public const MODE_REPLACE = 'replace';
|
public const MODE_REPLACE = 'replace';
|
||||||
@@ -36,9 +36,19 @@ class RestoreService {
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private IRootFolder $root,
|
private IRootFolder $root,
|
||||||
|
private LeaseService $leases,
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Shared lock on the location while restoring: a cleanup (exclusive) must not delete what is being read. */
|
||||||
|
private function withSharedLock(Target $target, callable $fn, int $ttl = 600): mixed {
|
||||||
|
try {
|
||||||
|
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, $fn);
|
||||||
|
} catch (\OCP\Lock\LockedException) {
|
||||||
|
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is running; try again later');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Direct children (files and folders) of a path in a snapshot.
|
* Direct children (files and folders) of a path in a snapshot.
|
||||||
*
|
*
|
||||||
@@ -72,7 +82,24 @@ class RestoreService {
|
|||||||
* @return array{restored:int, trashed:int, target:string}
|
* @return array{restored:int, trashed:int, target:string}
|
||||||
*/
|
*/
|
||||||
public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array {
|
public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array {
|
||||||
if (!preg_match('#^data/([^/]+)/files(?:/(.*))?$#', trim($logicalPath, '/'), $m)) {
|
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress) {
|
||||||
|
$last = time();
|
||||||
|
// Keep the lease alive while files are being written.
|
||||||
|
$beat = function (int $done, int $total, string $name) use ($refresh, $progress, &$last): void {
|
||||||
|
if (time() - $last >= 60) {
|
||||||
|
$refresh();
|
||||||
|
$last = time();
|
||||||
|
}
|
||||||
|
if ($progress !== null) {
|
||||||
|
$progress($done, $total, $name);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress): array {
|
||||||
|
if (!Repository::isSafePath($logicalPath) || !preg_match('#^data/([^/]+)/files(?:/(.*))?$#', $logicalPath, $m)) {
|
||||||
throw new \InvalidArgumentException('Expected a path like data/<user>/files/<folder or file>');
|
throw new \InvalidArgumentException('Expected a path like data/<user>/files/<folder or file>');
|
||||||
}
|
}
|
||||||
[$uid, $rel] = [$m[1], trim($m[2] ?? '', '/')];
|
[$uid, $rel] = [$m[1], trim($m[2] ?? '', '/')];
|
||||||
@@ -122,12 +149,30 @@ class RestoreService {
|
|||||||
return ['restored' => $restored, 'trashed' => $trashed, 'target' => $uid . ':' . $userFolder->getRelativePath($dest->getPath())];
|
return ['restored' => $restored, 'trashed' => $trashed, 'target' => $uid . ':' . $userFolder->getRelativePath($dest->getPath())];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Raw restore of any snapshot path to a local directory (administrators, disaster recovery). */
|
/**
|
||||||
public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory): int {
|
* Raw restore of any snapshot path to a local directory (administrators, disaster recovery).
|
||||||
|
*
|
||||||
|
* @param bool $stripPrefix write into $directory itself instead of $directory/$prefix (see
|
||||||
|
* Repository::restore()); used to restore "data/" or "config/" straight into the
|
||||||
|
* real data/config directory for occ backup:restore:full.
|
||||||
|
*/
|
||||||
|
public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null): int {
|
||||||
|
if ($prefix !== '' && !Repository::isSafePath($prefix)) {
|
||||||
|
throw new \InvalidArgumentException('Invalid path: ' . $prefix);
|
||||||
|
}
|
||||||
if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) {
|
if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) {
|
||||||
throw new \RuntimeException("Cannot create $directory");
|
throw new \RuntimeException("Cannot create $directory");
|
||||||
}
|
}
|
||||||
return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/');
|
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat): int {
|
||||||
|
$own = LeaseService::throttled($refresh);
|
||||||
|
$beat = static function () use ($own, $heartbeat): void {
|
||||||
|
$own();
|
||||||
|
if ($heartbeat !== null) {
|
||||||
|
$heartbeat();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat);
|
||||||
|
}, 3600);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void {
|
private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void {
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which snapshots to keep: the newest $last, plus the newest snapshot of each of the most recent
|
||||||
|
* $daily days, $weekly ISO weeks and $monthly months that have a snapshot (like restic/borg).
|
||||||
|
*/
|
||||||
|
final class RetentionPolicy {
|
||||||
|
public function __construct(
|
||||||
|
public readonly int $last = 3,
|
||||||
|
public readonly int $daily = 7,
|
||||||
|
public readonly int $weekly = 4,
|
||||||
|
public readonly int $monthly = 6,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static function fromArray(array $a): self {
|
||||||
|
return new self((int)($a['last'] ?? 3), (int)($a['daily'] ?? 7), (int)($a['weekly'] ?? 4), (int)($a['monthly'] ?? 6));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function toArray(): array {
|
||||||
|
return ['last' => $this->last, 'daily' => $this->daily, 'weekly' => $this->weekly, 'monthly' => $this->monthly];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param array<string, int> $snapshots id => unix time
|
||||||
|
* @return array<string, list<string>> id => reasons it is kept (ids not in the result are removed)
|
||||||
|
*/
|
||||||
|
public function keep(array $snapshots): array {
|
||||||
|
arsort($snapshots);
|
||||||
|
$keep = [];
|
||||||
|
$i = 0;
|
||||||
|
foreach ($snapshots as $id => $time) {
|
||||||
|
if ($i++ < $this->last) {
|
||||||
|
$keep[$id][] = 'last';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foreach (['daily' => 'Y-m-d', 'weekly' => 'o-W', 'monthly' => 'Y-m'] as $kind => $format) {
|
||||||
|
$seen = [];
|
||||||
|
foreach ($snapshots as $id => $time) {
|
||||||
|
$bucket = date($format, $time);
|
||||||
|
if (isset($seen[$bucket])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (count($seen) >= $this->$kind) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
$seen[$bucket] = true;
|
||||||
|
$keep[$id][] = $kind;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $keep;
|
||||||
|
}
|
||||||
|
}
|
||||||
+210
-19
@@ -10,10 +10,14 @@ namespace OCA\NgBackup\Service;
|
|||||||
use OCA\NgBackup\Backend\ExternalStorageFactory;
|
use OCA\NgBackup\Backend\ExternalStorageFactory;
|
||||||
use OCA\NgBackup\Backend\IBackend;
|
use OCA\NgBackup\Backend\IBackend;
|
||||||
use OCA\NgBackup\Backend\StorageBackend;
|
use OCA\NgBackup\Backend\StorageBackend;
|
||||||
|
use OCA\NgBackup\Db\Snapshot;
|
||||||
|
use OCA\NgBackup\Db\SnapshotMapper;
|
||||||
use OCA\NgBackup\Db\Target;
|
use OCA\NgBackup\Db\Target;
|
||||||
use OCA\NgBackup\Db\TargetMapper;
|
use OCA\NgBackup\Db\TargetMapper;
|
||||||
use OCA\NgBackup\Repository\Repository;
|
use OCA\NgBackup\Repository\Repository;
|
||||||
use OCP\App\IAppManager;
|
use OCP\App\IAppManager;
|
||||||
|
use OCP\AppFramework\Db\DoesNotExistException;
|
||||||
|
use OCP\Files\Storage\IStorage;
|
||||||
use OCP\Security\ICrypto;
|
use OCP\Security\ICrypto;
|
||||||
use OCP\Server;
|
use OCP\Server;
|
||||||
|
|
||||||
@@ -22,17 +26,19 @@ use OCP\Server;
|
|||||||
* object is built only while it is used; nothing is mounted. files_external is loaded for its
|
* object is built only while it is used; nothing is mounted. files_external is loaded for its
|
||||||
* classes but its enabled/disabled state is left as the admin set it.
|
* classes but its enabled/disabled state is left as the admin set it.
|
||||||
*/
|
*/
|
||||||
class TargetService {
|
final class TargetService {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetMapper $mapper,
|
private TargetMapper $mapper,
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
private ICrypto $crypto,
|
private ICrypto $crypto,
|
||||||
private IAppManager $appManager,
|
private IAppManager $appManager,
|
||||||
private \OCA\NgBackup\Db\DbIndexCache $indexCache,
|
private \OCA\NgBackup\Db\DbIndexCache $indexCache,
|
||||||
|
private \OCA\NgBackup\Db\AppConfigCatalogAnchor $anchor,
|
||||||
|
private SnapshotMapper $snapshots,
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<array{id:string, name:string, auth:list<string>, parameters:array}> */
|
/** @return list<array{id:string, name:string, auth:list<array{id:string, name:string, parameters:array}>, parameters:array}> */
|
||||||
public function availableBackends(): array {
|
public function availableBackends(): array {
|
||||||
return $this->factory()->describeBackends();
|
return $this->factory()->describeBackends();
|
||||||
}
|
}
|
||||||
@@ -45,21 +51,15 @@ class TargetService {
|
|||||||
* @return array{target: Target, created: bool}
|
* @return array{target: Target, created: bool}
|
||||||
*/
|
*/
|
||||||
public function add(string $name, string $backend, string $auth, array $options, string $basePath): array {
|
public function add(string $name, string $backend, string $auth, array $options, string $basePath): array {
|
||||||
if (!preg_match('/^[A-Za-z0-9][A-Za-z0-9 _.-]{0,63}$/', $name)) {
|
$this->validateName($name);
|
||||||
throw new \InvalidArgumentException('Name: letters, digits, space, _ . - (max 64)');
|
$this->requireUsableBackend($backend);
|
||||||
}
|
|
||||||
try {
|
|
||||||
$this->mapper->findByName($name);
|
|
||||||
throw new \InvalidArgumentException("A target named '$name' already exists");
|
|
||||||
} catch (\OCP\AppFramework\Db\DoesNotExistException) {
|
|
||||||
}
|
|
||||||
$keys = $this->keys->keyRing();
|
$keys = $this->keys->keyRing();
|
||||||
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath);
|
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath);
|
||||||
if ($storageBackend->exists('config')) {
|
if ($storageBackend->exists('config')) {
|
||||||
$repo = Repository::openWithKey($storageBackend, $keys);
|
$repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor);
|
||||||
$created = false;
|
$created = false;
|
||||||
} else {
|
} else {
|
||||||
$repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey());
|
$repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey(), $this->anchor);
|
||||||
$created = true;
|
$created = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,7 +77,159 @@ class TargetService {
|
|||||||
$target->setBasePath(trim($basePath, '/'));
|
$target->setBasePath(trim($basePath, '/'));
|
||||||
$target->setRepositoryId($repo->id());
|
$target->setRepositoryId($repo->id());
|
||||||
$target->setCreatedAt(time());
|
$target->setCreatedAt(time());
|
||||||
return ['target' => $this->mapper->insert($target), 'created' => $created];
|
$target = $this->mapper->insert($target);
|
||||||
|
if (!$created) {
|
||||||
|
// Disaster recovery: occ backup:list reads this local cache, not the repository
|
||||||
|
// directly, so a snapshot made by another installation must be backfilled here too.
|
||||||
|
$this->adoptSnapshots($target, $repo);
|
||||||
|
}
|
||||||
|
return ['target' => $target, 'created' => $created];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Change how an existing location is reached (credentials, host, folder, even the connection
|
||||||
|
* type) or its name. The new settings must lead to the same repository: the history stays
|
||||||
|
* and a typo cannot silently start a new, empty repository. A null option value keeps the
|
||||||
|
* stored one, so the web interface never has to send stored passwords back.
|
||||||
|
*
|
||||||
|
* @param array<string, mixed> $options
|
||||||
|
*/
|
||||||
|
public function update(Target $target, string $name, string $backend, string $auth, array $options, string $basePath): Target {
|
||||||
|
$this->validateName($name, $target->getId());
|
||||||
|
$this->requireUsableBackend($backend);
|
||||||
|
$stored = $this->options($target);
|
||||||
|
foreach ($options as $key => $value) {
|
||||||
|
if ($value === null) {
|
||||||
|
if (array_key_exists($key, $stored)) {
|
||||||
|
$options[$key] = $stored[$key];
|
||||||
|
} else {
|
||||||
|
unset($options[$key]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Do not create the folder: a typo in the path must not leave an empty folder behind.
|
||||||
|
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath, false);
|
||||||
|
if (!$storageBackend->exists('config')) {
|
||||||
|
if (!$storageBackend->canConnect()) {
|
||||||
|
throw new \InvalidArgumentException('Cannot connect with these settings. Check the host, user name and password');
|
||||||
|
}
|
||||||
|
throw new \InvalidArgumentException("No NG Backup repository found in folder '" . trim($basePath, '/') . "'. To start an empty repository, add a new location instead");
|
||||||
|
}
|
||||||
|
$repo = Repository::openWithKey($storageBackend, $this->keys->keyRing(), $this->indexCache, $this->anchor);
|
||||||
|
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
||||||
|
throw new \InvalidArgumentException('These settings lead to a different repository than the one of this location');
|
||||||
|
}
|
||||||
|
$target->setName($name);
|
||||||
|
$target->setBackend($backend);
|
||||||
|
$target->setAuth($auth);
|
||||||
|
$target->setOptions($this->crypto->encrypt(json_encode($options, JSON_THROW_ON_ERROR)));
|
||||||
|
$target->setBasePath(trim($basePath, '/'));
|
||||||
|
return $this->mapper->update($target);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The stored options for an edit form: password-type values are replaced by null, which
|
||||||
|
* update() reads as "keep the stored value".
|
||||||
|
*
|
||||||
|
* @return array<string, mixed>
|
||||||
|
*/
|
||||||
|
public function editableOptions(Target $target): array {
|
||||||
|
$secret = [];
|
||||||
|
foreach ($this->availableBackends() as $b) {
|
||||||
|
if ($b['id'] !== $target->getBackend()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$params = $b['parameters'];
|
||||||
|
foreach ($b['auth'] as $a) {
|
||||||
|
if ($a['id'] === $target->getAuth()) {
|
||||||
|
$params += $a['parameters'];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foreach ($params as $key => $p) {
|
||||||
|
if (($p['type'] ?? 0) === 2) {
|
||||||
|
$secret[$key] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$options = $this->options($target);
|
||||||
|
foreach ($options as $key => $value) {
|
||||||
|
if (isset($secret[$key])) {
|
||||||
|
$options[$key] = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $options;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function validateName(string $name, ?int $ownId = null): void {
|
||||||
|
if (!preg_match('/^[A-Za-z0-9][A-Za-z0-9 _.-]{0,63}$/', $name)) {
|
||||||
|
throw new \InvalidArgumentException('Name: letters, digits, space, _ . - (max 64)');
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$other = $this->mapper->findByName($name);
|
||||||
|
} catch (DoesNotExistException) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ($other->getId() !== $ownId) {
|
||||||
|
throw new \InvalidArgumentException("A target named '$name' already exists");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function requireUsableBackend(string $backend): void {
|
||||||
|
// Without the native extension files_external falls back to the smbclient binary, which
|
||||||
|
// registers fine but cannot seek: the first backup then fails with "Seek failed". Refuse
|
||||||
|
// up front instead of letting the admin find out from a failed run.
|
||||||
|
if ($backend === 'smb' && !extension_loaded('smbclient')) {
|
||||||
|
throw new \InvalidArgumentException('The SMB backend needs the PHP smbclient extension (pecl install smbclient); the smbclient binary alone cannot seek and every backup would fail');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A target's options are encrypted with Nextcloud's own instance secret (ICrypto), not
|
||||||
|
* ng_backup's key, so changing that secret (occ backup:restore:full merging config.php)
|
||||||
|
* would otherwise leave every target permanently undecryptable. Call $applySecretChange
|
||||||
|
* (which must be the only thing that changes the secret) through here instead: every
|
||||||
|
* target's options are decrypted first, then re-encrypted under whatever secret is active
|
||||||
|
* once $applySecretChange returns.
|
||||||
|
*/
|
||||||
|
public function reencryptOptionsAround(callable $applySecretChange): mixed {
|
||||||
|
$plain = [];
|
||||||
|
foreach ($this->mapper->findAll() as $target) {
|
||||||
|
$plain[$target->getId()] = $this->options($target);
|
||||||
|
}
|
||||||
|
$result = $applySecretChange();
|
||||||
|
foreach ($this->mapper->findAll() as $target) {
|
||||||
|
if (!isset($plain[$target->getId()])) {
|
||||||
|
continue; // added by $applySecretChange itself; nothing to re-encrypt
|
||||||
|
}
|
||||||
|
$target->setOptions($this->crypto->encrypt(json_encode($plain[$target->getId()], JSON_THROW_ON_ERROR)));
|
||||||
|
$this->mapper->update($target);
|
||||||
|
}
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function adoptSnapshots(Target $target, Repository $repo): void {
|
||||||
|
foreach ($repo->listSnapshots() as $snapshotId) {
|
||||||
|
try {
|
||||||
|
$this->snapshots->findOne($target->getId(), $snapshotId);
|
||||||
|
continue; // already known locally
|
||||||
|
} catch (DoesNotExistException) {
|
||||||
|
}
|
||||||
|
// One unreadable/corrupt snapshot must not stop the rest of the location's
|
||||||
|
// snapshots from being backfilled, nor abort adding the target itself.
|
||||||
|
try {
|
||||||
|
$meta = $repo->snapshotMeta($snapshotId);
|
||||||
|
$s = new Snapshot();
|
||||||
|
$s->setTargetId($target->getId());
|
||||||
|
$s->setSnapshotId($snapshotId);
|
||||||
|
$s->setKind((string)($meta['kind'] ?? 'full'));
|
||||||
|
$s->setLabel(($meta['label'] ?? '') !== '' ? $meta['label'] : null);
|
||||||
|
$s->setCreatedAt((int)strtotime((string)($meta['time'] ?? 'now')));
|
||||||
|
$s->setFiles((int)($meta['stats']['files'] ?? 0));
|
||||||
|
$s->setBytes((int)($meta['stats']['bytes'] ?? 0));
|
||||||
|
$this->snapshots->insert($s);
|
||||||
|
} catch (\Throwable) {
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<Target> */
|
/** @return list<Target> */
|
||||||
@@ -86,7 +238,22 @@ class TargetService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function get(string $nameOrId): Target {
|
public function get(string $nameOrId): Target {
|
||||||
|
try {
|
||||||
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
||||||
|
} catch (DoesNotExistException) {
|
||||||
|
throw new \InvalidArgumentException("No such location: $nameOrId");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Size limit for the data stored on this location (null = none). */
|
||||||
|
public function setMaxBytes(Target $target, ?int $maxBytes): Target {
|
||||||
|
$target->setMaxBytes($maxBytes);
|
||||||
|
return $this->mapper->update($target);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function setAppendOnly(Target $target, bool $appendOnly): Target {
|
||||||
|
$target->setAppendOnly($appendOnly);
|
||||||
|
return $this->mapper->update($target);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Remove the target from NG Backup. The backups on the target itself are not touched. */
|
/** Remove the target from NG Backup. The backups on the target itself are not touched. */
|
||||||
@@ -99,10 +266,11 @@ class TargetService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function repository(Target $target): Repository {
|
public function repository(Target $target): Repository {
|
||||||
$repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache);
|
$repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache, $this->anchor);
|
||||||
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
||||||
throw new \RuntimeException('The target now contains a different repository than when it was added');
|
throw new \RuntimeException('The target now contains a different repository than when it was added');
|
||||||
}
|
}
|
||||||
|
$repo->setMaxBytes($target->getMaxBytes());
|
||||||
return $repo;
|
return $repo;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -111,15 +279,38 @@ class TargetService {
|
|||||||
return json_decode($this->crypto->decrypt($target->getOptions()), true, 512, JSON_THROW_ON_ERROR);
|
return json_decode($this->crypto->decrypt($target->getOptions()), true, 512, JSON_THROW_ON_ERROR);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function buildBackend(string $backend, string $auth, array $options, string $basePath): StorageBackend {
|
private function buildBackend(string $backend, string $auth, array $options, string $basePath, bool $createBase = true): StorageBackend {
|
||||||
return new StorageBackend($this->factory()->create($backend, $auth, $options), $basePath);
|
$create = fn (): IStorage => $this->factory()->create($backend, $auth, $options);
|
||||||
|
return new StorageBackend($create(), $basePath, $createBase, $create);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private ?ExternalStorageFactory $factory = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* files_external's connection code, without changing the app's state:
|
||||||
|
* - enabled: it is booted by Nextcloud; use its own BackendService;
|
||||||
|
* - disabled: do NOT boot it (booting registers its mount provider and listeners, which use
|
||||||
|
* its database tables; on installations where it was never enabled those do not exist).
|
||||||
|
* Only its classes are loaded, and a private BackendService gets its backends and auth
|
||||||
|
* mechanisms from its Application class.
|
||||||
|
*/
|
||||||
private function factory(): ExternalStorageFactory {
|
private function factory(): ExternalStorageFactory {
|
||||||
// Load files_external's classes without enabling the app (its state stays as the admin set it).
|
if ($this->factory !== null) {
|
||||||
if (!class_exists(\OCA\Files_External\Service\BackendService::class)) {
|
return $this->factory;
|
||||||
|
}
|
||||||
|
if ($this->appManager->isEnabledForAnyone('files_external')) {
|
||||||
$this->appManager->loadApp('files_external');
|
$this->appManager->loadApp('files_external');
|
||||||
|
return $this->factory = new ExternalStorageFactory(Server::get(\OCA\Files_External\Service\BackendService::class));
|
||||||
}
|
}
|
||||||
return new ExternalStorageFactory(Server::get(\OCA\Files_External\Service\BackendService::class));
|
if (!class_exists(\OCA\Files_External\AppInfo\Application::class)) {
|
||||||
|
$path = $this->appManager->getAppPath('files_external');
|
||||||
|
require_once $path . '/composer/autoload.php';
|
||||||
|
}
|
||||||
|
$service = new \OCA\Files_External\Service\BackendService(
|
||||||
|
Server::get(\OCP\IAppConfig::class), Server::get(\Psr\Log\LoggerInterface::class));
|
||||||
|
$app = new \OCA\Files_External\AppInfo\Application();
|
||||||
|
$service->registerBackendProvider($app);
|
||||||
|
$service->registerAuthMechanismProvider($app);
|
||||||
|
return $this->factory = new ExternalStorageFactory($service);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Db\Target;
|
||||||
|
use OCA\NgBackup\UserMigration\RepositoryExportDestination;
|
||||||
|
use OCA\NgBackup\UserMigration\UserMigrationServiceLocator;
|
||||||
|
use OCP\IUser;
|
||||||
|
use OCP\Lock\LockedException;
|
||||||
|
use OCP\UserMigration\UserMigrationException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-user backup via user_migration: every registered migrator's export streams straight into
|
||||||
|
* the encrypted repository (no zip, no copy in the user's own storage, see
|
||||||
|
* RepositoryExportDestination). Shares blobs with the full backup; prune() keeps a user export's
|
||||||
|
* blobs referenced through its manifest under users/<uid>/.
|
||||||
|
*
|
||||||
|
* user_migration is an optional dependency (unlike files_external): without it, only this
|
||||||
|
* per-user backup/restore is unavailable, the full backup still works.
|
||||||
|
*/
|
||||||
|
final class UserBackupService {
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private LeaseService $leases,
|
||||||
|
private UserMigrationServiceLocator $locator,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @throws UserMigrationException when user_migration is not enabled, or the export itself fails */
|
||||||
|
public function backupUser(Target $target, IUser $user): string {
|
||||||
|
$service = $this->locator->get();
|
||||||
|
return $this->withSharedLock($target, function (callable $refresh) use ($target, $user, $service) {
|
||||||
|
$repo = $this->targets->repository($target);
|
||||||
|
$dest = new RepositoryExportDestination($repo, $user->getUID(), LeaseService::throttled($refresh));
|
||||||
|
$service->export($dest, $user);
|
||||||
|
// A short exclusive lock around the catalog write only: the shared lock above
|
||||||
|
// allows another concurrent backup (full or per-user) to be writing the catalog at
|
||||||
|
// the same time. The repository is reopened inside the lock, so the write builds on
|
||||||
|
// the latest catalog generation instead of the one loaded before the lock was taken.
|
||||||
|
$this->withCatalogLock($target, fn () => $this->targets->repository($target)->recordUserExport($dest->manifestPath()));
|
||||||
|
return $dest->manifestPath();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private function withSharedLock(Target $target, callable $fn): mixed {
|
||||||
|
try {
|
||||||
|
// Refreshed by RepositoryExportDestination as it writes data, so a long export keeps it.
|
||||||
|
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 3600, $fn);
|
||||||
|
} catch (LockedException) {
|
||||||
|
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function withCatalogLock(Target $target, callable $fn): mixed {
|
||||||
|
try {
|
||||||
|
return $this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300, $fn);
|
||||||
|
} catch (LockedException) {
|
||||||
|
throw new \RuntimeException('Another catalog write for ' . $target->getName() . ' is in progress; try again');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Db\Target;
|
||||||
|
use OCA\NgBackup\UserMigration\RepositoryExportDestination;
|
||||||
|
use OCA\NgBackup\UserMigration\RepositoryImportSource;
|
||||||
|
use OCA\NgBackup\UserMigration\UserMigrationServiceLocator;
|
||||||
|
use OCP\EventDispatcher\IEventDispatcher;
|
||||||
|
use OCP\Files\Events\Node\FilesystemTornDownEvent;
|
||||||
|
use OCP\IUser;
|
||||||
|
use OCP\IUserManager;
|
||||||
|
use OCP\Lock\LockedException;
|
||||||
|
use OCP\Security\ISecureRandom;
|
||||||
|
use OCP\UserMigration\UserMigrationException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restoring a user from a user_migration export (see UserBackupService), never over an existing
|
||||||
|
* account:
|
||||||
|
* - replace: a safety export of the current state goes into the repository first, then the
|
||||||
|
* account is deleted and recreated fresh from the backup, under the same uid;
|
||||||
|
* - as-backup: the backup is restored into a new account "<uid>-bak" (or "-bak-2", ...) instead,
|
||||||
|
* so an admin can compare and transfer by hand. Nextcloud cannot rename a uid, so the *backup*
|
||||||
|
* gets the new name, never the existing account.
|
||||||
|
*/
|
||||||
|
final class UserRestoreService {
|
||||||
|
public const MODE_REPLACE = 'replace';
|
||||||
|
public const MODE_AS_BACKUP = 'as-backup';
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private LeaseService $leases,
|
||||||
|
private UserMigrationServiceLocator $locator,
|
||||||
|
private IUserManager $userManager,
|
||||||
|
private ISecureRandom $random,
|
||||||
|
private IEventDispatcher $events,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @throws UserMigrationException when user_migration is not enabled, or the restore itself fails */
|
||||||
|
public function restoreUser(Target $target, string $manifestPath, string $mode = self::MODE_REPLACE): IUser {
|
||||||
|
$service = $this->locator->get();
|
||||||
|
return $this->withSharedLock($target, function (callable $refresh) use ($target, $manifestPath, $mode, $service): IUser {
|
||||||
|
$repo = $this->targets->repository($target);
|
||||||
|
// Only ever read a manifest the verified catalog actually recorded: a path alone
|
||||||
|
// (even one this app generated before) proves nothing about it still being current
|
||||||
|
// or genuine once the catalog's rollback protection is supposed to vouch for it.
|
||||||
|
if (!$repo->isRecordedUserExport($manifestPath)) {
|
||||||
|
throw new \InvalidArgumentException("$manifestPath is not a recorded user export for this location");
|
||||||
|
}
|
||||||
|
$beat = LeaseService::throttled($refresh);
|
||||||
|
$source = new RepositoryImportSource($repo, $manifestPath, $beat);
|
||||||
|
$originalUid = $source->getOriginalUid();
|
||||||
|
|
||||||
|
if ($mode === self::MODE_AS_BACKUP) {
|
||||||
|
$backupUid = $this->freeBackupUid($originalUid);
|
||||||
|
$user = $this->userManager->createUser($backupUid, $this->random->generate(32));
|
||||||
|
if ($user === false) {
|
||||||
|
throw new \RuntimeException("Could not create the account $backupUid");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return $service->import($source, $user);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
// A partially-imported, never-used account left behind is strictly worse
|
||||||
|
// than no account at all; the original source account was never touched.
|
||||||
|
$user->delete();
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($mode !== self::MODE_REPLACE) {
|
||||||
|
throw new \InvalidArgumentException("Unknown restore mode: $mode");
|
||||||
|
}
|
||||||
|
$existing = $this->userManager->get($originalUid);
|
||||||
|
$safetyManifestPath = null;
|
||||||
|
if ($existing !== null) {
|
||||||
|
// Never import over a live account: export its current state first, so nothing
|
||||||
|
// is lost if the restore turns out to be the wrong choice.
|
||||||
|
$safety = new RepositoryExportDestination($repo, $originalUid, $beat);
|
||||||
|
$service->export($safety, $existing);
|
||||||
|
$safetyManifestPath = $safety->manifestPath();
|
||||||
|
// Same race as UserBackupService::backupUser(): reopen the repository inside the
|
||||||
|
// catalog lock so the write builds on the latest generation.
|
||||||
|
$this->withCatalogLock($target, fn () => $this->targets->repository($target)->recordUserExport($safetyManifestPath));
|
||||||
|
if (!$existing->delete()) {
|
||||||
|
throw new \RuntimeException("Could not delete the existing account $originalUid before restoring; its safety export is at $safetyManifestPath");
|
||||||
|
}
|
||||||
|
// IRootFolder caches a Folder per uid for the lifetime of this process; without
|
||||||
|
// this it would still hand the just-deleted account's (now storage-less) folder
|
||||||
|
// to the import below, and every write would fail.
|
||||||
|
$this->events->dispatchTyped(new FilesystemTornDownEvent());
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return $service->import($source, null);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
// The original account is already gone at this point (or never existed); a
|
||||||
|
// failure here can only be reported, not undone in place. Its safety export
|
||||||
|
// (if any) is the recovery path, already recorded above.
|
||||||
|
throw new UserMigrationException(
|
||||||
|
"Could not restore $originalUid" . ($safetyManifestPath !== null ? "; its safety export is at $safetyManifestPath" : '') . ': ' . $e->getMessage(),
|
||||||
|
0,
|
||||||
|
$e,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private function freeBackupUid(string $uid): string {
|
||||||
|
$candidate = $uid . '-bak';
|
||||||
|
for ($n = 2; $this->userManager->userExists($candidate); $n++) {
|
||||||
|
$candidate = $uid . '-bak-' . $n;
|
||||||
|
}
|
||||||
|
return $candidate;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function withSharedLock(Target $target, callable $fn): mixed {
|
||||||
|
try {
|
||||||
|
// Refreshed while the safety export writes and the import reads (see UserBackupService::withSharedLock()).
|
||||||
|
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 3600, $fn);
|
||||||
|
} catch (LockedException) {
|
||||||
|
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function withCatalogLock(Target $target, callable $fn): mixed {
|
||||||
|
try {
|
||||||
|
return $this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300, $fn);
|
||||||
|
} catch (LockedException) {
|
||||||
|
throw new \RuntimeException('Another catalog write for ' . $target->getName() . ' is in progress; try again');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Service;
|
||||||
|
|
||||||
|
use OCA\NgBackup\AppInfo\Application;
|
||||||
|
use OCA\NgBackup\Db\Target;
|
||||||
|
use OCP\IAppConfig;
|
||||||
|
use OCP\Lock\LockedException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checksum audit of a snapshot (occ backup:verify), see Repository::verify(). On a large
|
||||||
|
* repository a check takes hours (a deep one days), so how far it is gets stored every
|
||||||
|
* PROGRESS_SECONDS, together with how it ended: `occ backup:status` shows it.
|
||||||
|
*/
|
||||||
|
final class VerifyService {
|
||||||
|
public const PROGRESS_SECONDS = 30;
|
||||||
|
private const KEY_PREFIX = 'verify_progress_';
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private TargetService $targets,
|
||||||
|
private LeaseService $leases,
|
||||||
|
private IAppConfig $appConfig,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param ?callable(array):void $onProgress called with the stored progress every PROGRESS_SECONDS
|
||||||
|
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
||||||
|
*/
|
||||||
|
public function verify(Target $target, string $snapshotId, bool $deep = false, ?callable $onProgress = null): array {
|
||||||
|
return $this->run($target, $snapshotId, $deep, $onProgress, fn (\OCA\NgBackup\Repository\Repository $repo, callable $heartbeat): array => $repo->verify($snapshotId, $deep, $heartbeat));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param ?callable(array):void $onProgress called with the stored progress every PROGRESS_SECONDS
|
||||||
|
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
||||||
|
*/
|
||||||
|
public function verifyUserExport(Target $target, string $manifestPath, bool $deep = false, ?callable $onProgress = null): array {
|
||||||
|
return $this->run($target, $manifestPath, $deep, $onProgress, fn (\OCA\NgBackup\Repository\Repository $repo, callable $heartbeat): array => $repo->verifyUserExport($manifestPath, $deep, $heartbeat));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One line on a check's progress or outcome, for occ (see progress()). */
|
||||||
|
public static function describe(array $s): string {
|
||||||
|
$size = static fn (int $b): string => $b >= 1073741824 ? sprintf('%.1f GiB', (float)$b / 1073741824.0) : sprintf('%.1f MiB', (float)$b / 1048576.0);
|
||||||
|
$count = static fn (int $done, mixed $total): string => number_format($done)
|
||||||
|
. (is_int($total) && $total > 0 ? '/' . number_format($total) . sprintf(' (%d%%)', intdiv(100 * $done, $total)) : '');
|
||||||
|
$deep = ($s['deep'] ?? false) === true;
|
||||||
|
$kind = $deep ? 'deep check' : 'check';
|
||||||
|
if (($s['status'] ?? '') === 'running') {
|
||||||
|
$phase = match ($s['phase'] ?? '') {
|
||||||
|
'index' => 'index files ' . $count((int)$s['done'], $s['total'] ?? null),
|
||||||
|
'packs' => 'pack folders ' . $count((int)$s['done'], $s['total'] ?? null),
|
||||||
|
'files', 'database' => 'files ' . $count((int)$s['done'], $s['total'] ?? null) . ', ' . number_format((int)($s['blobs'] ?? 0)) . ' blobs'
|
||||||
|
. ($deep ? ', ' . $size((int)($s['bytes'] ?? 0)) . ' decrypted' : ''),
|
||||||
|
default => 'starting',
|
||||||
|
};
|
||||||
|
$age = time() - (int)($s['updated'] ?? 0);
|
||||||
|
return sprintf('%s running since %s: %s; updated %s ago%s', $kind, date('Y-m-d H:i', (int)$s['started']), $phase,
|
||||||
|
$age < 120 ? $age . ' s' : intdiv($age, 60) . ' min',
|
||||||
|
$age > 600 ? ' (no progress for a while: the process may have stopped)' : '');
|
||||||
|
}
|
||||||
|
$took = gmdate('H:i:s', max(0, (int)($s['finished'] ?? 0) - (int)($s['started'] ?? 0)));
|
||||||
|
return match ($s['status'] ?? '') {
|
||||||
|
'ok' => sprintf('%s OK on %s (took %s): %s files, %s blobs%s', $kind, date('Y-m-d H:i', (int)$s['finished']), $took,
|
||||||
|
number_format((int)($s['files'] ?? 0)), number_format((int)($s['blobs'] ?? 0)), $deep ? ', ' . $size((int)($s['bytes'] ?? 0)) . ' decrypted' : ''),
|
||||||
|
'problems' => sprintf('%s found PROBLEMS on %s: %d missing, %d failed, %d unreadable blob(s); run occ backup:verify again for the details',
|
||||||
|
$kind, date('Y-m-d H:i', (int)$s['finished']), (int)($s['missing'] ?? 0), (int)($s['failedBlobs'] ?? 0), (int)($s['unreadable'] ?? 0)),
|
||||||
|
default => sprintf('%s FAILED on %s: %s', $kind, date('Y-m-d H:i', (int)($s['finished'] ?? 0)), (string)($s['error'] ?? 'unknown error')),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The stored state of the last check per location: running (with how far it is) or how it
|
||||||
|
* ended.
|
||||||
|
*
|
||||||
|
* @return array<int, array<string, mixed>> by location id
|
||||||
|
*/
|
||||||
|
public function progress(): array {
|
||||||
|
$result = [];
|
||||||
|
foreach ($this->appConfig->getKeys(Application::APP_ID) as $key) {
|
||||||
|
if (!str_starts_with($key, self::KEY_PREFIX)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$state = json_decode($this->appConfig->getValueString(Application::APP_ID, $key, '', lazy: true), true);
|
||||||
|
if (is_array($state)) {
|
||||||
|
$result[(int)substr($key, strlen(self::KEY_PREFIX))] = $state;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ksort($result);
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function run(Target $target, string $what, bool $deep, ?callable $onProgress, callable $fn): array {
|
||||||
|
$ttl = $deep ? 600 : 300;
|
||||||
|
$state = ['what' => $what, 'deep' => $deep, 'status' => 'running', 'started' => time(), 'updated' => time(),
|
||||||
|
'pid' => getmypid(), 'phase' => 'starting', 'done' => 0, 'total' => null, 'blobs' => 0, 'bytes' => 0];
|
||||||
|
try {
|
||||||
|
$this->store($target, $state);
|
||||||
|
} catch (\Throwable) {
|
||||||
|
// progress reports are a convenience; the check itself goes ahead
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$result = $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, function (callable $refresh) use ($target, $ttl, $fn, $onProgress, &$state): array {
|
||||||
|
$lastRefresh = time();
|
||||||
|
$lastStore = time();
|
||||||
|
$heartbeat = function (?array $progress = null) use ($refresh, $ttl, $target, $onProgress, &$lastRefresh, &$lastStore, &$state): void {
|
||||||
|
$now = time();
|
||||||
|
if ($now - $lastRefresh >= intdiv($ttl, 2)) {
|
||||||
|
$refresh();
|
||||||
|
$lastRefresh = $now;
|
||||||
|
}
|
||||||
|
if ($progress !== null && $now - $lastStore >= self::PROGRESS_SECONDS) {
|
||||||
|
$state = array_merge($state, $progress, ['updated' => $now]);
|
||||||
|
try {
|
||||||
|
$this->store($target, $state);
|
||||||
|
} catch (\Throwable) {
|
||||||
|
// a progress report that cannot be stored must not stop the check
|
||||||
|
}
|
||||||
|
$lastStore = $now;
|
||||||
|
if ($onProgress !== null) {
|
||||||
|
$onProgress($state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return $fn($this->targets->repository($target), $heartbeat);
|
||||||
|
});
|
||||||
|
} catch (LockedException) {
|
||||||
|
$this->finish($target, $state, 'failed', ['error' => 'A cleanup of the location was in progress']);
|
||||||
|
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
$this->finish($target, $state, 'failed', ['error' => $e->getMessage()]);
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
$problems = count($result['missing']) + count($result['failed']) + count($result['unreadable']);
|
||||||
|
$this->finish($target, $state, $problems === 0 ? 'ok' : 'problems', [
|
||||||
|
'phase' => 'done', 'files' => $result['filesChecked'], 'blobs' => $result['blobsChecked'], 'bytes' => $result['bytesChecked'],
|
||||||
|
'missing' => count($result['missing']), 'failedBlobs' => count($result['failed']), 'unreadable' => count($result['unreadable']),
|
||||||
|
]);
|
||||||
|
return $result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function finish(Target $target, array $state, string $status, array $extra): void {
|
||||||
|
try {
|
||||||
|
$this->store($target, array_merge($state, $extra, ['status' => $status, 'updated' => time(), 'finished' => time()]));
|
||||||
|
} catch (\Throwable) {
|
||||||
|
// the check's own result counts; its stored summary is a convenience
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function store(Target $target, array $state): void {
|
||||||
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_PREFIX . $target->getId(), json_encode($state, JSON_THROW_ON_ERROR), lazy: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Settings;
|
||||||
|
|
||||||
|
use OCP\AppFramework\Http\TemplateResponse;
|
||||||
|
use OCP\Settings\ISettings;
|
||||||
|
use OCP\Util;
|
||||||
|
|
||||||
|
final class Admin implements ISettings {
|
||||||
|
public function getForm(): TemplateResponse {
|
||||||
|
Util::addScript('ng_backup', 'ng_backup-admin');
|
||||||
|
Util::addStyle('ng_backup', 'ng_backup-admin');
|
||||||
|
return new TemplateResponse('ng_backup', 'admin');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getSection(): string {
|
||||||
|
return 'ng_backup';
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getPriority(): int {
|
||||||
|
return 10;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\Settings;
|
||||||
|
|
||||||
|
use OCP\IL10N;
|
||||||
|
use OCP\IURLGenerator;
|
||||||
|
use OCP\Settings\IIconSection;
|
||||||
|
|
||||||
|
final class AdminSection implements IIconSection {
|
||||||
|
public function __construct(
|
||||||
|
private IL10N $l,
|
||||||
|
private IURLGenerator $url,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getID(): string {
|
||||||
|
return 'ng_backup';
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getName(): string {
|
||||||
|
return $this->l->t('NG Backup');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getPriority(): int {
|
||||||
|
return 80;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getIcon(): string {
|
||||||
|
return $this->url->imagePath('ng_backup', 'app-dark.svg');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\SetupCheck;
|
||||||
|
|
||||||
|
use OCA\NgBackup\Db\Run;
|
||||||
|
use OCA\NgBackup\Db\RunMapper;
|
||||||
|
use OCA\NgBackup\Service\AlertService;
|
||||||
|
use OCA\NgBackup\Service\BackupService;
|
||||||
|
use OCA\NgBackup\Service\KeyService;
|
||||||
|
use OCA\NgBackup\Service\TargetService;
|
||||||
|
use OCP\IL10N;
|
||||||
|
use OCP\IURLGenerator;
|
||||||
|
use OCP\SetupCheck\ISetupCheck;
|
||||||
|
use OCP\SetupCheck\SetupResult;
|
||||||
|
|
||||||
|
/** Shows the backup state in the admin overview (Security & setup warnings). */
|
||||||
|
final class BackupStatusCheck implements ISetupCheck {
|
||||||
|
public function __construct(
|
||||||
|
private IL10N $l,
|
||||||
|
private IURLGenerator $url,
|
||||||
|
private KeyService $keys,
|
||||||
|
private TargetService $targets,
|
||||||
|
private RunMapper $runs,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getCategory(): string {
|
||||||
|
return 'system';
|
||||||
|
}
|
||||||
|
|
||||||
|
public function getName(): string {
|
||||||
|
return $this->l->t('NG Backup');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function run(): SetupResult {
|
||||||
|
$link = $this->url->linkToRouteAbsolute('settings.AdminSettings.index', ['section' => 'ng_backup']);
|
||||||
|
if (!$this->keys->isInitialized() || !$this->keys->everConfirmed()) {
|
||||||
|
return SetupResult::error($this->l->t('Backups are not set up: create the backup key and confirm the recovery kit.'), $link);
|
||||||
|
}
|
||||||
|
if ($this->keys->recoveryKitConfirmation() === null) {
|
||||||
|
return SetupResult::error($this->l->t('The backup passphrase was changed: download and confirm the new recovery kit.'), $link);
|
||||||
|
}
|
||||||
|
$targets = $this->targets->list();
|
||||||
|
if ($targets === []) {
|
||||||
|
return SetupResult::warning($this->l->t('No backup location is configured.'), $link);
|
||||||
|
}
|
||||||
|
$problems = [];
|
||||||
|
foreach ($targets as $t) {
|
||||||
|
$last = $this->runs->lastFinished($t->getId(), BackupService::KIND_FULL);
|
||||||
|
if ($last === null) {
|
||||||
|
$problems[] = $this->l->t('%s: no successful backup yet', [$t->getName()]);
|
||||||
|
} elseif (time() - (int)$last->getFinishedAt() > AlertService::STALE_AFTER) {
|
||||||
|
$problems[] = $this->l->t('%1$s: last successful backup %2$s', [$t->getName(), date('Y-m-d H:i', (int)$last->getFinishedAt())]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foreach ($this->runs->findRecent(10) as $r) {
|
||||||
|
if ($r->getStatus() === Run::FAILED && $r->getStartedAt() > time() - AlertService::STALE_AFTER) {
|
||||||
|
$problems[] = $this->l->t('A %1$s run failed: %2$s', [$r->getKind(), mb_strimwidth((string)$r->getError(), 0, 120, '…')]);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $problems === []
|
||||||
|
? SetupResult::success($this->l->t('Recent backups to all locations succeeded.'))
|
||||||
|
: SetupResult::warning(implode("\n", $problems), $link);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -28,6 +28,8 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private Repository $repo,
|
private Repository $repo,
|
||||||
private string $uid,
|
private string $uid,
|
||||||
|
/** Called while data is written (per chunk), to keep the location lease alive. */
|
||||||
|
private ?\Closure $heartbeat = null,
|
||||||
) {
|
) {
|
||||||
$this->exportId = gmdate('Ymd\THis\Z') . '-' . bin2hex(random_bytes(4));
|
$this->exportId = gmdate('Ymd\THis\Z') . '-' . bin2hex(random_bytes(4));
|
||||||
}
|
}
|
||||||
@@ -36,6 +38,7 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
$writer = $this->repo->blobWriter($this->stats);
|
$writer = $this->repo->blobWriter($this->stats);
|
||||||
$writer->write($content);
|
$writer->write($content);
|
||||||
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => strlen($content)];
|
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => strlen($content)];
|
||||||
|
$this->beat();
|
||||||
}
|
}
|
||||||
|
|
||||||
public function addFileAsStream(string $path, $stream): void {
|
public function addFileAsStream(string $path, $stream): void {
|
||||||
@@ -46,6 +49,7 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
throw new UserMigrationException('Read error while exporting ' . $path);
|
throw new UserMigrationException('Read error while exporting ' . $path);
|
||||||
}
|
}
|
||||||
$writer->write($chunk);
|
$writer->write($chunk);
|
||||||
|
$this->beat();
|
||||||
}
|
}
|
||||||
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => $writer->bytes()];
|
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => $writer->bytes()];
|
||||||
}
|
}
|
||||||
@@ -65,11 +69,14 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
if ($stream === false) {
|
if ($stream === false) {
|
||||||
throw new \RuntimeException('Cannot read ' . $node->getPath());
|
throw new \RuntimeException('Cannot read ' . $node->getPath());
|
||||||
}
|
}
|
||||||
|
try {
|
||||||
$this->addFileAsStream($target, $stream);
|
$this->addFileAsStream($target, $stream);
|
||||||
|
} finally {
|
||||||
fclose($stream);
|
fclose($stream);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public function setMigratorVersions(array $versions): void {
|
public function setMigratorVersions(array $versions): void {
|
||||||
$this->versions = $versions;
|
$this->versions = $versions;
|
||||||
@@ -89,6 +96,12 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
return $this->stats;
|
return $this->stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function beat(): void {
|
||||||
|
if ($this->heartbeat !== null) {
|
||||||
|
($this->heartbeat)();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static function norm(string $path): string {
|
private static function norm(string $path): string {
|
||||||
return trim($path, '/');
|
return trim($path, '/');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ declare(strict_types=1);
|
|||||||
namespace OCA\NgBackup\UserMigration;
|
namespace OCA\NgBackup\UserMigration;
|
||||||
|
|
||||||
use OCA\NgBackup\Repository\Repository;
|
use OCA\NgBackup\Repository\Repository;
|
||||||
|
use OCP\Files\File;
|
||||||
use OCP\Files\Folder;
|
use OCP\Files\Folder;
|
||||||
use OCP\UserMigration\IImportSource;
|
use OCP\UserMigration\IImportSource;
|
||||||
use OCP\UserMigration\UserMigrationException;
|
use OCP\UserMigration\UserMigrationException;
|
||||||
@@ -19,6 +20,8 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private Repository $repo,
|
private Repository $repo,
|
||||||
string $manifestPath,
|
string $manifestPath,
|
||||||
|
/** Called while blobs are read, to keep the location lease alive. */
|
||||||
|
private ?\Closure $heartbeat = null,
|
||||||
) {
|
) {
|
||||||
$this->manifest = json_decode($repo->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
$this->manifest = json_decode($repo->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
||||||
}
|
}
|
||||||
@@ -30,11 +33,18 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @return resource */
|
||||||
public function getFileAsStream(string $path) {
|
public function getFileAsStream(string $path) {
|
||||||
$entry = $this->entry($path, 'f');
|
$entry = $this->entry($path, 'f');
|
||||||
// Phase 0: buffer in php://temp (memory up to 8 MiB, then a temp file for this one file).
|
// Phase 0: buffer in php://temp (memory up to 8 MiB, then a temp file for this one file).
|
||||||
$stream = fopen('php://temp/maxmemory:' . (8 * 1048576), 'w+b');
|
$stream = fopen('php://temp/maxmemory:' . (8 * 1048576), 'w+b');
|
||||||
|
if ($stream === false) {
|
||||||
|
throw new UserMigrationException('Cannot buffer ' . $path);
|
||||||
|
}
|
||||||
foreach ($this->repo->readBlobs($entry['b']) as $data) {
|
foreach ($this->repo->readBlobs($entry['b']) as $data) {
|
||||||
|
if ($this->heartbeat !== null) {
|
||||||
|
($this->heartbeat)();
|
||||||
|
}
|
||||||
for ($done = 0, $len = strlen($data); $done < $len; $done += $n) {
|
for ($done = 0, $len = strlen($data); $done < $len; $done += $n) {
|
||||||
$n = fwrite($stream, $done === 0 ? $data : substr($data, $done));
|
$n = fwrite($stream, $done === 0 ? $data : substr($data, $done));
|
||||||
if ($n === false || $n === 0) {
|
if ($n === false || $n === 0) {
|
||||||
@@ -51,6 +61,7 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
$prefix = trim($path, '/');
|
$prefix = trim($path, '/');
|
||||||
$prefix = $prefix === '' ? '' : $prefix . '/';
|
$prefix = $prefix === '' ? '' : $prefix . '/';
|
||||||
$names = [];
|
$names = [];
|
||||||
|
/** @var string $p */
|
||||||
foreach (array_keys($this->manifest['entries']) as $p) {
|
foreach (array_keys($this->manifest['entries']) as $p) {
|
||||||
if ($prefix === '' || str_starts_with($p, $prefix)) {
|
if ($prefix === '' || str_starts_with($p, $prefix)) {
|
||||||
$rest = substr($p, strlen($prefix));
|
$rest = substr($p, strlen($prefix));
|
||||||
@@ -70,6 +81,7 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
if (isset($this->manifest['entries'][$p])) {
|
if (isset($this->manifest['entries'][$p])) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
/** @var string $k */
|
||||||
foreach (array_keys($this->manifest['entries']) as $k) {
|
foreach (array_keys($this->manifest['entries']) as $k) {
|
||||||
if (str_starts_with($k, $p . '/')) {
|
if (str_starts_with($k, $p . '/')) {
|
||||||
return true;
|
return true;
|
||||||
@@ -91,17 +103,29 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
$this->copyToFolder($sub, $path);
|
$this->copyToFolder($sub, $path);
|
||||||
} else {
|
} else {
|
||||||
$stream = $this->getFileAsStream($path);
|
$stream = $this->getFileAsStream($path);
|
||||||
|
try {
|
||||||
if ($destination->nodeExists($name)) {
|
if ($destination->nodeExists($name)) {
|
||||||
$destination->get($name)->putContent($stream);
|
$existing = $destination->get($name);
|
||||||
|
if (!$existing instanceof File) {
|
||||||
|
throw new UserMigrationException("$path exists and is not a file");
|
||||||
|
}
|
||||||
|
$existing->putContent($stream);
|
||||||
} else {
|
} else {
|
||||||
$destination->newFile($name, $stream);
|
$destination->newFile($name, $stream);
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
// putContent()/newFile() already close the stream once they've consumed it:
|
||||||
|
// despite the @return resource docblock above, $stream is no longer a
|
||||||
|
// resource by the time we get here (verified against a real storage; a
|
||||||
|
// double fclose() throws a TypeError on PHP 8).
|
||||||
|
/** @psalm-suppress RedundantConditionGivenDocblockType */
|
||||||
if (is_resource($stream)) {
|
if (is_resource($stream)) {
|
||||||
fclose($stream);
|
fclose($stream);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public function getMigratorVersions(): array {
|
public function getMigratorVersions(): array {
|
||||||
return $this->manifest['versions'];
|
return $this->manifest['versions'];
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\UserMigration;
|
||||||
|
|
||||||
|
use OCP\App\AppPathNotFoundException;
|
||||||
|
use OCP\App\IAppManager;
|
||||||
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
use Symfony\Component\Console\Helper\QuestionHelper;
|
||||||
|
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Installs or enables the optional user_migration app, but only when the administrator asked for it
|
||||||
|
* (occ prompt or --install-user-migration). Runs occ in a child process, as the admin would.
|
||||||
|
*/
|
||||||
|
final class UserMigrationInstaller {
|
||||||
|
private const APP = 'user_migration';
|
||||||
|
|
||||||
|
public function __construct(
|
||||||
|
private IAppManager $appManager,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
public function isEnabled(): bool {
|
||||||
|
return $this->appManager->isEnabledForAnyone(self::APP);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function codeIsPresent(): bool {
|
||||||
|
try {
|
||||||
|
$this->appManager->getAppPath(self::APP);
|
||||||
|
return true;
|
||||||
|
} catch (AppPathNotFoundException) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Installs (from the app store, if missing) and enables user_migration; throws on failure. */
|
||||||
|
public function install(): void {
|
||||||
|
$args = $this->codeIsPresent() ? ['app:enable', self::APP] : ['app:install', self::APP];
|
||||||
|
/** @psalm-suppress UndefinedClass */
|
||||||
|
$occ = \OC::$SERVERROOT . '/occ';
|
||||||
|
$command = implode(' ', array_map('escapeshellarg', [PHP_BINARY, $occ, ...$args])) . ' 2>&1';
|
||||||
|
exec($command, $lines, $code);
|
||||||
|
if ($code !== 0) {
|
||||||
|
throw new \RuntimeException("occ " . implode(' ', $args) . " failed:\n" . implode("\n", $lines));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns when user_migration is enabled. Otherwise asks the administrator (interactive shell) or
|
||||||
|
* requires --install-user-migration, and throws when neither gives consent.
|
||||||
|
*
|
||||||
|
* @throws \RuntimeException
|
||||||
|
*/
|
||||||
|
public function ensureEnabled(InputInterface $input, OutputInterface $output): void {
|
||||||
|
if ($this->isEnabled()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$consent = $input->getOption('install-user-migration') === true;
|
||||||
|
if (!$consent && $input->isInteractive()) {
|
||||||
|
$consent = (new QuestionHelper())->ask($input, $output, new ConfirmationQuestion(
|
||||||
|
'The user_migration app is not enabled. Install and enable it now? [y/N] ', false)) === true;
|
||||||
|
}
|
||||||
|
if (!$consent) {
|
||||||
|
throw new \RuntimeException('The user_migration app is not enabled; per-user backup/restore needs it. Run "occ '
|
||||||
|
. 'app:install ' . self::APP . '" and "occ app:enable ' . self::APP . '", or repeat with --install-user-migration');
|
||||||
|
}
|
||||||
|
$output->writeln('Installing and enabling user_migration (approved by the administrator)...');
|
||||||
|
$this->install();
|
||||||
|
throw new \RuntimeException('user_migration is installed and enabled; run this command again');
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
namespace OCA\NgBackup\UserMigration;
|
||||||
|
|
||||||
|
use OCP\App\IAppManager;
|
||||||
|
use OCP\Server;
|
||||||
|
use OCP\UserMigration\UserMigrationException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolves user_migration's own (internal, non-public) UserMigrationService on demand.
|
||||||
|
*
|
||||||
|
* Unlike files_external, user_migration is an optional dependency: without it, only per-user
|
||||||
|
* backup/restore is unavailable, the full backup still works. The class is never autoloaded by
|
||||||
|
* composer; everything that touches it is behind this one class.
|
||||||
|
*/
|
||||||
|
final class UserMigrationServiceLocator {
|
||||||
|
public function __construct(
|
||||||
|
private IAppManager $appManager,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @throws UserMigrationException when user_migration is not enabled anywhere */
|
||||||
|
public function get(): object {
|
||||||
|
if (!$this->appManager->isEnabledForAnyone('user_migration')) {
|
||||||
|
throw new UserMigrationException('The user_migration app is not enabled; per-user backup/restore is unavailable');
|
||||||
|
}
|
||||||
|
$this->appManager->loadApp('user_migration');
|
||||||
|
return Server::get(\OCA\UserMigration\Service\UserMigrationService::class);
|
||||||
|
}
|
||||||
|
}
|
||||||
Generated
+8381
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user