Compare commits
153 Commits
v1.0.0-rc.1
..
1.1.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 6ae1b2277f | |||
| 2a72acb70b | |||
| fb8e8d0694 | |||
| 042fdae686 | |||
| 99f1655d04 | |||
| 373d7e0424 | |||
| c5024d3ddc | |||
| 345b1d545e | |||
| 98067f7746 | |||
| cd951af1a1 | |||
| cf9cf50286 | |||
| 2b21596b3a | |||
| 953312bf44 | |||
| 569077fa69 | |||
| 51a4a67bc4 | |||
| a8b5e85c9b | |||
| 36f2427e9a | |||
| e58b86fc48 | |||
| 193f65940b | |||
| 6540a9314a | |||
| f81c5e13e9 | |||
| 1b621b7149 | |||
| 1736005648 | |||
| c7a651f269 | |||
| 8897d8074f | |||
| 0b9fa1ae92 | |||
| ee406c481e | |||
| dd2e1af997 | |||
| d96229f370 | |||
| b14b16c66c | |||
| ce40053cec | |||
| 46e0186f35 | |||
| d5eb7abbff | |||
| 4c973d5cd8 | |||
| e9592cb6c6 | |||
| b86b07a49c | |||
| ecae6e9cdd | |||
| d36c40e77b | |||
| 50c2636926 | |||
| d1e9a53ce5 | |||
| b625fe7666 | |||
| 74bfa7052f | |||
| bf48d339e4 | |||
| 029b3b9cc7 | |||
| dce93aa75e | |||
| 3ec23a3072 | |||
| 4a395c75f2 | |||
| 36a30732ad | |||
| d7c7ca093b | |||
| 05d02f3c70 | |||
| dfe1fe9aad | |||
| f0293e6d01 | |||
| f7c9894b78 | |||
| be3746e1c3 | |||
| f5473495ce | |||
| 3694184674 | |||
| a0fc91a700 | |||
| a11e6c7254 | |||
| 6f1a37f4f4 | |||
| 5cde57bc1b | |||
| 3eec2e9554 | |||
| cdf0b70e25 | |||
| 66615c2a88 | |||
| cc75c7128c | |||
| a0c4abb60a | |||
| 4847ce293d | |||
| 90dd97977a | |||
| 08534975d7 | |||
| 95e3dfcf6d | |||
| 7f4cae0e42 | |||
| 965440d1f7 | |||
| 3a0f882a38 | |||
| 632d2a2838 | |||
| 9e9234e2a8 | |||
| 3ac3298313 | |||
| 384b2d2792 | |||
| 8032411789 | |||
| b4f20cb070 | |||
| 343389b98b | |||
| 133f5f9d34 | |||
| cf4bd9db7f | |||
| 67cc350b8c | |||
| fb5b234e8a | |||
| fd12c586b6 | |||
| 6ffe979ea7 | |||
| a03b7354b3 | |||
| be1c628f07 | |||
| a8355df12e | |||
| f0537f0f2c | |||
| fd0f786f5f | |||
| 2eabdf44d5 | |||
| 61e2a28b67 | |||
| 7ed53ad0eb | |||
| bdcb4f0cae | |||
| 2616b6e543 | |||
| 9bc4e4a4ab | |||
| 827e46381e | |||
| 8eb6c4c033 | |||
| d623416306 | |||
| cd251cffb3 | |||
| 0b4872599d | |||
| 95184dadc7 | |||
| 9b6bd7d2ab | |||
| 2fcd38f478 | |||
| b7d09c9784 | |||
| 177361acae | |||
| 24686897c3 | |||
| 85ea0acc50 | |||
| ac57e31c93 | |||
| 315e83b847 | |||
| e461c48056 | |||
| d504067cd8 | |||
| c37b26ea38 | |||
| b728db36c7 | |||
| 6ed0a555cf | |||
| 221ed9dc17 | |||
| 285153a968 | |||
| 6c8a78f6bb | |||
| 0accd5a3f2 | |||
| 583592f69b | |||
| a384ab41bb | |||
| 9b85221e50 | |||
| c95f552cc6 | |||
| 6121f684e9 | |||
| 43a71684f9 | |||
| 88b719879d | |||
| 1d7f7e6423 | |||
| ccbbf01c54 | |||
| b789d17636 | |||
| 8e5c8ccda4 | |||
| e0b79939f9 | |||
| 39b4fe359b | |||
| 40e147f3a0 | |||
| 7f6e5c15f8 | |||
| 11ac30635a | |||
| 9e1378c6ab | |||
| 04e91359a0 | |||
| 46eaceb928 | |||
| 9eeaefd938 | |||
| 4f6a2e5b70 | |||
| 7b735ea941 | |||
| d8ef50732f | |||
| 5af712acc3 | |||
| 01e16199f7 | |||
| 0924da48ac | |||
| 88477a1a9f | |||
| f633de59b4 | |||
| 523620acde | |||
| bbddc83f80 | |||
| 456dffc880 | |||
| 4d423825a0 | |||
| 29a56e432b | |||
| 7f29d009f3 |
@@ -0,0 +1 @@
|
||||
custom: ["https://www.ensembia.com"]
|
||||
@@ -41,6 +41,20 @@ jobs:
|
||||
- run: composer install --no-interaction --no-progress
|
||||
- run: composer psalm
|
||||
|
||||
frontend:
|
||||
# Builds the admin page, so a broken Vue component fails CI instead of showing up after a release.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
- run: npm ci --no-audit --no-fund
|
||||
- run: npm run build
|
||||
|
||||
integration:
|
||||
# Full round trip (key, kit, location, backup, incremental, restore modes, database dump and
|
||||
# restore, retention, trash) on a fresh Nextcloud per version and database.
|
||||
@@ -104,3 +118,27 @@ jobs:
|
||||
persist-credentials: false
|
||||
- name: Full restore round trip
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache
|
||||
- name: Full restore round trip on PostgreSQL, with Mail (foreign keys) and a disabled app
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql
|
||||
- name: Full restore round trip on MariaDB, with Mail (foreign keys) and a disabled app
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache mysql
|
||||
- name: Full restore of a backup made by 1.0.0-rc.1 (no table structure) on PostgreSQL
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql legacy
|
||||
- name: Full restore of a backup made by 1.0.0-rc.1 (no table structure) on MariaDB
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache mysql legacy
|
||||
- name: Full restore over SFTP on PostgreSQL, reading packs ahead on several connections
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql sftp
|
||||
- name: Full restore of a MariaDB backup onto PostgreSQL (ID counters after the data)
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache mysql to-pgsql
|
||||
- name: Full restore of a PostgreSQL backup onto MariaDB
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql to-mysql
|
||||
- name: Full restore of a PostgreSQL backup onto SQLite
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache pgsql to-sqlite
|
||||
- name: Full restore of a MariaDB backup onto SQLite
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache mysql to-sqlite
|
||||
- name: Full restore of a SQLite backup onto PostgreSQL
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite to-pgsql
|
||||
- name: Full restore with a damaged database dump (--salvage)
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite salvage
|
||||
- name: Full restore with a damaged and a missing pack (--salvage)
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache sqlite salvage-files
|
||||
|
||||
@@ -7,9 +7,327 @@ the app store accepts a `-beta` suffix, but Nextcloud servers then only offer th
|
||||
beta/daily update channel, which would hide the app from most admins. The stability stage is
|
||||
instead carried by the app description, the git tag, the release, and this file.
|
||||
|
||||
## [Unreleased]
|
||||
Each heading starts with that `<version>`, once: the app store shows a release the section(s)
|
||||
whose heading starts with its `x.y.z` and ignores anything after it, so `## [1.0.0-rc.6]` would be
|
||||
shown under 1.0.0. The git tag of a release candidate goes after the date.
|
||||
|
||||
## [1.0.0-rc.1] - 2026-10-17
|
||||
## [1.1.0] - 2026-11-09
|
||||
|
||||
### Added
|
||||
- Pin a restore point, on the restore page or with `occ backup:pin`: the retention policy leaves it
|
||||
alone, for example the one from before an upgrade. Unpinning notifies all administrators.
|
||||
- A note on a restore point that you can change afterwards, on the restore page or with
|
||||
`occ backup:note`. Pins and notes are stored encrypted on the location, so a new server sees them.
|
||||
An older version of NG Backup can still read the location, but drops them when it writes to it.
|
||||
- A schedule per location: off, once a day, or every N hours from a given time, on the page under
|
||||
Backups or with `occ backup:target:schedule`. After the update every location keeps the time that
|
||||
held for all of them; a new location starts with the schedule of the first one. A schedule is
|
||||
shown and changed in the time zone it was saved in, which the page names next to the time.
|
||||
- All folders get their original modification time back on a restore, not only the empty ones, from
|
||||
the first backup made with this version. An older version of NG Backup can still read new restore
|
||||
points. A restore through Nextcloud into a folder that is already there leaves its time alone.
|
||||
- Heavy background jobs (Memories, Recognize, Preview Generator, full text search, the antivirus
|
||||
background scan) pause while a backup runs, at most 6 hours in a row, then they get an hour. No app
|
||||
is turned off, and a job that is running goes on. On by default; turn it off under Backups or with
|
||||
`occ backup:pause-jobs off`.
|
||||
- Disaster recovery without the recovery kit: on a new server without a backup key,
|
||||
`occ backup:target:add` finds the backups on the location and takes their key from there with one
|
||||
of the old server's passphrases. Backups then wait until a new recovery kit is confirmed. The
|
||||
deletion delay is now also stored on the location, sealed with the key, from its first backup
|
||||
with this version; a location backed up only by an older version gives the default of 7 days.
|
||||
When the settings on the location cannot be read, the longest delay (365 days) is taken, so
|
||||
nothing leaves the Bin sooner than on the old server; the command says so.
|
||||
- A passphrase change that cannot reach a location (offline at the time) now reaches it at the
|
||||
location's next backup, so a new server can open it with the new passphrase.
|
||||
- `occ backup:target:add --repair-config`: when the config file of a location is gone, the recovery
|
||||
kit writes it again (for locations backed up by this version or newer). Before, the message said
|
||||
this was possible, but it was not.
|
||||
- The key page warns against creating a new key when you are restoring a server. A key made there
|
||||
by mistake can still make way for the key of the backups, as long as it was never used (no
|
||||
location, no backup): `occ backup:key:import --replace-unused-key` with the kit, or
|
||||
`occ backup:target:add --replace-unused-key` with a passphrase of the old server. A key that
|
||||
protected a location is never replaced, and neither is a key from before this version. All
|
||||
administrators are notified of a replacement.
|
||||
|
||||
### Changed
|
||||
- The list of backups stays within the page: a long label, note or error wraps onto more lines
|
||||
instead of making the table scroll sideways.
|
||||
- Reading files for a backup is much faster on disks where every request waits, such as network and
|
||||
cloud volumes: files are read in pieces of 4 MiB instead of 8 KiB. On a test server the disk
|
||||
delivered 27 MiB/s instead of 5 MiB/s.
|
||||
- Locations are backed up one after the other: a backup that has to wait shows as queued. A restore
|
||||
point has the time its backup really began, not the time it was queued.
|
||||
- Choosing a time no longer starts a backup right away when that time has already passed today.
|
||||
- A location without a schedule no longer gets the notification that it has had no recent backup.
|
||||
Instead, all administrators are told when a schedule is turned off, and reminded once a week while
|
||||
another location is still backed up, or every day when none is.
|
||||
- A file or folder that is left out because of its name now gets a notification of its own, in the
|
||||
log and in `occ backup:run`. It says the item is missing from the backup, gives example paths and
|
||||
asks to rename it; before, it was reported as unreadable, with advice about permissions.
|
||||
|
||||
## [1.0.3] - 2026-11-06
|
||||
|
||||
### Fixed
|
||||
- Empty folders are backed up and restored (#27), in `occ backup:restore`, `occ backup:restore:full`
|
||||
and the restore page, where they are shown too. They come back from the first backup made with
|
||||
this version; older restore points do not contain them. An older version of NG Backup can still
|
||||
read new restore points, and then only leaves the empty folders out.
|
||||
- A full disk during a restore no longer looks like a damaged backup. The restore now stops with a
|
||||
message about the disk, also with `--salvage`, and does not point at `--salvage` any more. The same
|
||||
goes for other trouble writing to the restore directory, such as permissions.
|
||||
- A file or folder whose name a snapshot cannot hold (a backslash, a control character, not UTF-8) is
|
||||
left out and reported as unreadable. Before, a backslash or control character could make the whole
|
||||
restore point unreadable, and a name that is not UTF-8 made the backup fail.
|
||||
- A restore that goes on after a stop also removes the temporary files in folders where every file
|
||||
is already in place, and checks that those folders are inside the restore directory before it
|
||||
leaves their files alone.
|
||||
- After a restore that was killed and then ran again, a backup no longer has to wait up to two hours
|
||||
for the locks of the killed run.
|
||||
- A restore only removes read-ahead folders of its own (`.ngb-restore-spool-` with 8 hex digits).
|
||||
- A restore keeps its lock on the location during a long row of files that are already in place.
|
||||
- A database restore that fails no longer leaves behind tables it created, which could stop the
|
||||
installation of that app later. When only NG Backup's own settings cannot be put back after the
|
||||
database was restored, `--salvage` no longer says the database is still the fresh one.
|
||||
- A connection that breaks while opening a pack on the location (seen with SMB) is retried once and
|
||||
is handled by `--salvage`, instead of stopping the restore with the error of the storage.
|
||||
- Without a system cron, several running backups take turns while the page is open.
|
||||
- The Bin tab shows how far it is while it reads the removed restore points, which can take a while.
|
||||
It also shows when it cannot read a location, instead of an empty bin, and refreshes when a run
|
||||
ends.
|
||||
|
||||
## [1.0.2] - 2026-11-06
|
||||
|
||||
### Added
|
||||
- A warning on the admin page and in `occ backup:status` when background jobs do not run on a
|
||||
system cron, or the cron stopped, with a link to how to set one up. Backups, also the scheduled
|
||||
ones, need it (#13).
|
||||
- Without a working system cron, a backup started with "Back up now" moves on while the NG Backup
|
||||
page stays open (#13).
|
||||
- `--salvage` for `occ backup:restore` and `occ backup:restore:full`: restore whatever can be
|
||||
read when a backup is damaged. Files that cannot be read or decrypted are left out and listed,
|
||||
and the command exits with code 2. When the database cannot be restored, `backup:restore:full`
|
||||
leaves the fresh database as it is and puts the data directory and config of the backup in
|
||||
`ngb-salvage-<snapshot>` in the data directory, to put back by hand.
|
||||
- The trash of restore points has a tab of its own, "Bin", instead of being shown below the
|
||||
backups.
|
||||
|
||||
### Changed
|
||||
- A backup in its database phase shows how far it is, for example "running (db: 47/183 tables)",
|
||||
and one that has not moved for 15 minutes says "waiting for background jobs" with when it last
|
||||
took a step. Before, it only said "running (db)", which looked the same as being stuck (#13).
|
||||
- Full restores from S3 read the packs ahead, eight ranges at a time, as restores over SFTP do
|
||||
since 1.0.0-rc.4. Over SMB the packs are read in 8 MiB reads instead of 1 MiB: over a line with
|
||||
a 130 ms round trip, reading was four times faster in a test (18 instead of 4.5 MiB/s).
|
||||
- Over SMB the index files, which a new server reads in `backup:target:add`, take fewer round
|
||||
trips: from Canada to a Hetzner Storage Box (130 ms), 0.66 instead of 1.05 s per file, so about
|
||||
5 instead of 8 hours for the 27,000 index files of a 1 TB repository.
|
||||
- `occ backup:restore:full` that runs again after it stopped (`--force`) goes on where it was: data
|
||||
files already in place with the size and modification time of the backup are left, and only
|
||||
the rest is read from the location. Before, a stopped restore of 1 TB started over.
|
||||
|
||||
### Fixed
|
||||
- A full restore over SMB could hang at 100% CPU when the connection to the server dropped: a
|
||||
failed read was counted as data. It is now read again over a new connection. Found in a
|
||||
disaster recovery test of 1 TB from Canada to a Hetzner Storage Box, after 392 GiB.
|
||||
- The read-ahead folder of a restore that was killed while it wrote is removed by the next run;
|
||||
it kept one file of up to 8 MiB.
|
||||
- `occ backup:target:add` only estimates the time left for reading the index once enough of it
|
||||
is in. The first file also carries the time to connect, which made it say "about 642 min left"
|
||||
for something done within a minute.
|
||||
- A restore that stops because a file is gone from the location now points at `--salvage`, as it
|
||||
already did for damaged data.
|
||||
|
||||
### Known issues
|
||||
- Empty folders are not backed up or restored; to be fixed in 1.0.3 (#27).
|
||||
- A full disk during a restore looks like a damaged backup ("Size mismatch"), and with `--salvage`
|
||||
the restore goes on and lists the remaining files. The backup itself is fine: make room and run
|
||||
the restore again. To be fixed in 1.0.3.
|
||||
|
||||
## [1.0.1] - 2026-11-01
|
||||
|
||||
Fixes for what was reported after 1.0.0 (#22, #24) and small improvements. Nothing changes in
|
||||
your backups or settings.
|
||||
|
||||
### Added
|
||||
- `occ backup:user:list [location] [user]` shows the user exports on a location, with the manifest
|
||||
path `backup:user:restore` expects. Before, that path was only printed once by
|
||||
`backup:user:backup` (#22).
|
||||
- A "Docs" tab on the admin page, with links to the how-to, the disaster recovery guide and the
|
||||
roadmap. It also shows before the backup key is set up.
|
||||
- A label for a backup started with "Back up now" (optional; `occ backup:run -l` already had one).
|
||||
It shows with the restore point.
|
||||
- An explanation in the trash of restore points of what "removed by retention policy" means: the
|
||||
policy keeps the newest restore points and the last one of each recent day, week and month, and
|
||||
moves the others to the trash, where they can be brought back until they are deleted
|
||||
permanently. `occ backup:trash` says the same below its list.
|
||||
|
||||
### Changed
|
||||
- `occ backup:status`, `backup:list`, `backup:trash` and `backup:browse` show times in the time
|
||||
zone of the daily schedule and name it, as the admin page shows them in the administrator's.
|
||||
Before, they showed the server's UTC, which read as a wrong start time.
|
||||
- The admin page no longer says NG Backup is in beta.
|
||||
- `occ backup:user:restore` also takes a user id instead of the manifest path, when that user has
|
||||
exactly one export on the location. With more, and for a manifest path that is not on the
|
||||
location, the error lists that user's exports; for a user without exports it names the
|
||||
expected form (#22). Before, `backup:user:restore <location> <uid>` only said "is not a
|
||||
recorded user export for this location".
|
||||
|
||||
### Fixed
|
||||
- An SFTP location with an RSA key in OpenSSH format ("BEGIN OPENSSH PRIVATE KEY") stopped
|
||||
working after an upgrade to Nextcloud 35 with "Unable to read key", and such a key could not be
|
||||
added there at all (#24). The "Private key" field on the admin page is a single line, so a
|
||||
pasted key lost its line breaks; Nextcloud 34 still read it, Nextcloud 35 reads keys with a
|
||||
stricter library. The line breaks are now restored before the key is used, so a stored location
|
||||
works again without re-entering it.
|
||||
- A full restore onto SQLite could stop at once with "database is locked" when another process
|
||||
wrote to the database just as the restore began (a background job that was already running).
|
||||
The restore now takes the write lock first and waits for the other process instead. The
|
||||
full-restore test on SQLite runs with a second process writing alongside.
|
||||
- `occ backup:restore ... --to-directory` of a single file restored nothing ("Restored 0
|
||||
files"); only a folder worked. A path to a file now restores that file.
|
||||
- A restore that was stopped (a crash, a timeout, a restart) left its half-written temporary file
|
||||
(`<name>.ngb-restore-<id>`) next to the file it was restoring, and a `files:scan` then showed it
|
||||
in Files. A restore now removes such leftovers in each folder it writes to.
|
||||
- The app store showed the notes of every release candidate under 1.0.0, because their headings
|
||||
started with "1.0.0-rc". Each heading now starts with the app version of that release.
|
||||
|
||||
## [1.0.0] - 2026-10-29
|
||||
|
||||
First stable release. NG Backup makes encrypted, incremental backups of your whole Nextcloud
|
||||
(files, database and configuration) to S3, SFTP, WebDAV, SMB and other storage, and restores them
|
||||
from the web interface or with occ. Disaster recovery has been tested on real installations; see
|
||||
"Tested disaster recoveries" in the README, and read "Known limitations" before you rely on it.
|
||||
|
||||
The code is the same as 1.0.0-rc.6 (app version 0.9.12): updating from it changes nothing in your
|
||||
backups or settings.
|
||||
|
||||
## [0.9.12] - 2026-10-24 (1.0.0-rc.6)
|
||||
|
||||
Sixth release candidate for 1.0.0; the app version is 0.9.12. Fixes from two disaster recovery
|
||||
simulations and a user report.
|
||||
|
||||
### Fixed
|
||||
- The daily backup time was taken as UTC (Nextcloud runs PHP in UTC): 03:00 ran at 05:00 in
|
||||
Berlin (#19). The time is now saved with the administrator's time zone and shown with it. A time
|
||||
saved before is converted once to the zone of the server or an administrator, so it shows as it
|
||||
effectively ran (02:30 UTC becomes 04:30 Europe/Berlin in summer) and nothing moves.
|
||||
- A full restore over SMB stopped after 17 hours with "Decryption failed": a read over a long
|
||||
line handed out wrong bytes (the backup itself was fine, the same files restored identically
|
||||
afterwards). A blob that does not decrypt as first read, of a file or of the database, is now
|
||||
read once more on a request of its own, and only a second failure stops the restore.
|
||||
- A full restore onto MySQL/MariaDB stopped with "Incorrect string value" when it created a table
|
||||
(of an app that is not installed here) whose rows hold emoji or other 4-byte characters: the
|
||||
table got 3-byte utf8. Tables the restore creates now get this installation's character set and
|
||||
collation (utf8mb4). Found in a disaster recovery simulation onto MariaDB.
|
||||
|
||||
### Changed
|
||||
- The admin page uses more of the screen (sections up to 1400 pixels instead of 900), so the
|
||||
list of backup runs fits without scrolling sideways.
|
||||
- README: a roadmap, the tested disaster recoveries, measurements for OVH S3, the Storage Box over
|
||||
SFTP and SMB, how to build the SMB extension when PECL is down, and what works without shell
|
||||
access. The guide lists shell access among what you need.
|
||||
- The full-restore integration test checks a 4-byte character in a table the restore creates, and
|
||||
retries app installs from the app store.
|
||||
|
||||
## [0.9.11] - 2026-10-22 (1.0.0-rc.5)
|
||||
|
||||
Fifth release candidate for 1.0.0; the app version is 0.9.11. A full restore of 1 TB over SFTP
|
||||
stopped after 13 hours with "Blob not in index"; the backup itself was fine.
|
||||
|
||||
### Changed
|
||||
- README and app store description: NG Backup needs background jobs on system cron, with how to
|
||||
set that up (issues #13, #17).
|
||||
|
||||
### Fixed
|
||||
- A full restore no longer stops with "Blob not in index" after a pack could not be read for a
|
||||
moment. Reading the index again then got an empty listing (from a connection the location had
|
||||
dropped) and emptied the index. Reading the index again now only adds, an empty listing of a
|
||||
location that has an index is an error instead of an empty index, and a pack that could not be
|
||||
read is read once more before the restore gives up.
|
||||
- After a full restore of a MySQL/MariaDB or SQLite backup onto PostgreSQL, new files, users or
|
||||
activities could not be added ("duplicate key", "File entry could not be inserted"): such
|
||||
backups record no ID counters, and they stayed at 1. Every counter of a restored table now goes
|
||||
after its data. Found in a disaster recovery simulation of 138 GB from MariaDB to PostgreSQL,
|
||||
where all 15,772 files and every count were identical.
|
||||
- A full restore now runs in maintenance mode: a background job, a page visit or a user writing
|
||||
into the instance meanwhile could collide with restored rows ("duplicate key") or be lost. It
|
||||
is switched off when the restore completes; after an error it stays on, and the message gives
|
||||
the one line to switch it off and run the restore again. Maintenance mode that was on before
|
||||
the restore stays on.
|
||||
- Adding a location whose repository config could not be read, while backup data is there,
|
||||
no longer creates a new repository over it: it stops with an explanation. A snapshot that
|
||||
cannot be taken over when a location is added is now logged instead of left out silently.
|
||||
- An SFTP connection that a Hetzner Storage Box dropped without phpseclib noticing is now
|
||||
recognised and replaced: the check asks the server and counts its answer.
|
||||
- The extra connection a full restore reads ahead on is replaced when it breaks (up to 20 times),
|
||||
instead of reading the rest of the restore the slow way.
|
||||
|
||||
## [0.9.10] - 2026-10-19 (1.0.0-rc.4)
|
||||
|
||||
Fourth release candidate for 1.0.0; the app version is 0.9.10. 1.0.0-rc.3 (0.9.9) was not
|
||||
released: its fixes are in here.
|
||||
|
||||
### Changed
|
||||
- A full restore over SFTP reads the packs ahead on a connection of its own, 32 requests in
|
||||
flight, into temporary files in the restore folder. From a Hetzner Storage Box to a server in
|
||||
Canada (122 ms) packs now come in at about 14 MiB/s instead of 2.8. More connections can be set
|
||||
with `restore_sftp_connections` (README). A part that cannot be read this way is read as
|
||||
before. Restores from other locations work as before.
|
||||
- App store description: the line about Ensembia.
|
||||
|
||||
### Fixed
|
||||
- After a full restore stopped halfway, running it again with `--force` was refused when the
|
||||
database was already restored: Nextcloud then first wants `occ upgrade`. The guide and the
|
||||
error message now say so.
|
||||
- A full restore of a backup made before 1.0.0-rc.2 with `--skip-missing-tables` failed on
|
||||
PostgreSQL with "relation ... does not exist": the left-out tables were still looked up to set
|
||||
their ID counters. It now skips them. A new test restores a backup made by 1.0.0-rc.1 on
|
||||
PostgreSQL and MariaDB.
|
||||
- The app store showed old screenshots: it keeps a copy per URL, and the URLs always pointed at
|
||||
`main`. They now point at the release's tag.
|
||||
|
||||
## [0.9.8] - 2026-10-18 (1.0.0-rc.2)
|
||||
|
||||
Second release candidate for 1.0.0; the app version is 0.9.8. Fixes from a full disaster recovery
|
||||
test of a production instance (about 1 TB, Nextcloud All-in-One onto a plain Nextcloud with
|
||||
PostgreSQL).
|
||||
|
||||
### Added
|
||||
- `docs/DISASTER-RECOVERY.md`: step-by-step guide for when your server is gone.
|
||||
- `occ backup:restore:full --install-apps` installs (from the app store) or enables the apps the
|
||||
old server had enabled, before the restore starts.
|
||||
|
||||
### Changed
|
||||
- Backups record the structure of every database table. A full restore creates tables that do not
|
||||
exist on the new instance, for example those left behind by an app that was removed long ago or
|
||||
is switched off, instead of stopping at the first one.
|
||||
- A full restore no longer refuses when apps are missing: it restores their data and lists the
|
||||
apps to install afterwards. `nextcloud-aio` is never required, so a backup of Nextcloud
|
||||
All-in-One can be restored onto a plain Nextcloud.
|
||||
- A full restore onto an installation with a different table prefix (`dbtableprefix`) is refused
|
||||
before it starts, with what to change, instead of restoring into tables nobody uses.
|
||||
- All problems are listed at once, with the app each missing table probably belongs to. For
|
||||
backups made before this version, `--skip-missing-tables` leaves out tables whose structure is
|
||||
unknown.
|
||||
- A fuller app store description, ready for 1.0.0, and a note on paid restore help by Ensembia.
|
||||
- README: switching the server to TCP BBR made backups over a long distance about 3 times faster.
|
||||
|
||||
### Fixed
|
||||
- A full restore failed on foreign keys (for example those of the Mail app), because tables are
|
||||
restored in name order. Foreign keys are now checked after all rows are loaded, before the
|
||||
restore commits.
|
||||
- `occ backup:target:add` without a key pointed to `backup:key:init`, which makes a new key that
|
||||
cannot read existing backups. It now points to `backup:key:import` for a restore, and says that
|
||||
connecting to a location with backups can take a while. While it reads the index of a location
|
||||
with backups, it shows how far it is and about how long it still takes.
|
||||
- A restore stopped with "Not found: packs/..." when the location was cleaned up while it ran,
|
||||
for example by the weekly cleanup of the original server during a recovery test on a second
|
||||
server: blobs that were moved into new packs were still looked for in the old ones. A restore
|
||||
now reloads the index and reads them from where they are now.
|
||||
- Removing a location left its cached index in the database (tens of thousands of rows for a large
|
||||
backup). It is removed with the location now, and once for locations that are already gone.
|
||||
|
||||
## [0.9.7] - 2026-10-17 (1.0.0-rc.1)
|
||||
|
||||
Release candidate for 1.0.0; the app version is 0.9.7. Only bug fixes until 1.0.0. Tested on a
|
||||
production instance (about 1 TB, 150,000 files): full and incremental backups and a verify on
|
||||
@@ -288,7 +606,13 @@ and not implemented yet.
|
||||
- See the "Storage backend experiences" section of the README for provider-specific quirks found
|
||||
so far (e.g. Hetzner Storage Box sub-accounts).
|
||||
|
||||
[1.0.0-rc.1]: https://github.com/dillardblom/ng_backup/releases/tag/v1.0.0-rc.1
|
||||
[1.0.1]: https://github.com/dillardblom/ng_backup/releases/tag/v1.0.1
|
||||
[1.0.0]: https://github.com/dillardblom/ng_backup/releases/tag/v1.0.0
|
||||
[0.9.12]: https://github.com/dillardblom/ng_backup/releases/tag/v1.0.0-rc.6
|
||||
[0.9.11]: https://github.com/dillardblom/ng_backup/releases/tag/v1.0.0-rc.5
|
||||
[0.9.10]: https://github.com/dillardblom/ng_backup/releases/tag/v1.0.0-rc.4
|
||||
[0.9.8]: https://github.com/dillardblom/ng_backup/releases/tag/v1.0.0-rc.2
|
||||
[0.9.7]: https://github.com/dillardblom/ng_backup/releases/tag/v1.0.0-rc.1
|
||||
[0.9.6-beta.7]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.6-beta.7
|
||||
[0.9.5-beta.6]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.5-beta.6
|
||||
[0.9.4-beta.5]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.4-beta.5
|
||||
|
||||
@@ -383,11 +383,222 @@ in README "Known limitations":
|
||||
de gebruiker, wachtwoordzin min. 16 tekens). Geen nieuwe functies meer.
|
||||
- **Vóór 1.0.0 (verplicht):** volledige restore met kit op een tweede server (PVE02), alleen lezend
|
||||
of op een kopie van de locatie; plus een user-restore. Zonder geslaagde test geen 1.0.0.
|
||||
- **1.1.0:**
|
||||
- **1.0.0 (besluit 2026-10-18):** alleen betrouwbaarheid. rc2 uitgebracht; slaagt de grote
|
||||
DR-test op VM 980 (snapshot van 12-10, met `--skip-missing-tables`), dan 1.0.0. Zo niet: fix,
|
||||
rc3, en rc3 pas uitbrengen na een geslaagde volledige DR-test.
|
||||
- **rc4 (besluit 2026-10-19):** rc3 is niet uitgebracht. De DR-test met rc3 liep ~2,8 MiB/s
|
||||
(~3,5 dag); parallel lezen over SFTP haalde vanaf VM 980 ~7,6 MiB/s. Daarom rc4 = rc3 +
|
||||
vooruitlezen over SFTP (4 eigen verbindingen). Volgorde: volledige DR-test op VM 980 volgens de
|
||||
gids (SFTP, Storage Box) → rc4 uitbrengen en om tests vragen → zelf een volledige DR van een
|
||||
productie-instantie van ~250 GB via S3 en via SMB. Drie volledige DR-restores vóór 1.0.0.
|
||||
Problemen of meldingen van gebruikers: rc5.
|
||||
- **1.0.2** (plan akkoord 2026-11-04): punten 1, 2 en 5 hieronder, de Bin, nieuwe screenshots en
|
||||
het polish-punt over de schatting van `target:add`. Besluiten 2026-11-04:
|
||||
- `--salvage` haalt ook bij een kapotte database terug wat kan: bestanden zijn per gebruiker
|
||||
gescheiden en het waardevolst. Kan de database niet terug (dump onleesbaar, of past niet op
|
||||
deze installatie), dan blijft de verse database staan en gaan data en config van de backup
|
||||
naar `<datadir>/ngb-salvage-<snapshot>/`; vanaf daar is het handwerk. Exitcode 2.
|
||||
- De DR-test met een backup mét tabelstructuur, zonder `--skip-missing-tables` (punt 3), is
|
||||
nog niet gedaan en wordt ingepland vóór de release van 1.0.2. Ingepland (2026-11-04): een
|
||||
verse snapshot van nc01 (gemaakt met 1.0.1, dus met tabelstructuur) volgens de DR-gids
|
||||
terugzetten op VM 980 met de 1.0.2-build, over **SMB** (Storage Box; besluit 2026-11-04:
|
||||
SMB in plaats van SFTP, zodat de DR-test de nieuwe SMB-leescode op echte schaal en over de
|
||||
echte lijn vanuit Canada test; vorige meting 2,6 MiB/s, verwachting ~1 dag voor 1 TB), zonder
|
||||
`--skip-missing-tables`; daarna `--salvage` op een deel van de bestanden (moet hetzelfde
|
||||
resultaat geven, exitcode 0). Uitvoeren met Dillard, die VM 980 klaarzet; deze week, vóór
|
||||
de release. Screenshots akkoord (2026-11-04).
|
||||
**DR-test 1.0.2 over SMB (VM 980, snapshot a86271ed van nc01, 983 GiB, gestart 2026-11-04):**
|
||||
index lezen 8 uur (~1 bestand/s; daarna versneld naar 0,66 s/bestand, gemeten vanaf VM 980);
|
||||
restore zonder opties (geen `--install-apps`, geen `--skip-missing-tables`) liep ~12 MiB/s
|
||||
(SFTP eerder 2,8-7,6; SMB vóór 1.0.2 2,6). Na 392 GiB viel de SMB-verbinding weg en bleef de
|
||||
restore op 100% CPU hangen: de smbclient-extensie geeft bij een mislukte read een string met
|
||||
lengte -1. Opgelost (controle + nieuwe verbinding), plus hervatten: `--force` laat bestanden met
|
||||
dezelfde grootte en mtime staan. Hervat 2026-11-05 07:21 UTC na `maintenance:mode --off`
|
||||
volgens de gids; binnen 10 min voorbij de 392 GiB. SMB op sub5 na de test weer uitzetten.
|
||||
**Resultaat (klaar 2026-11-06 02:11 UTC, nazorg 02:27):** hervatte run 18 u 50 min, exit 0, 150.755
|
||||
bestanden; 7-13 MiB/s (langzamer vanaf ~18 uur Duitse tijd, oorzaak niet vastgesteld). Vergeleken
|
||||
met nc01: 150.467 van 150.758 bestanden gelijk in grootte en mtime, 397 op hash zonder verschil;
|
||||
gebruikers, agenda's, contacten, shares en Deck gelijk; bestanden, agenda en contacten via WebDAV
|
||||
getest als gebruiker. `backup:restore --salvage` op 16.904 bestanden: exit 0, geen verschil. Niet
|
||||
vastgesteld: de 50.080 "updated" bij `files:scan`. Zonder `--install-apps` 51 van de 81 apps aan.
|
||||
VM's 980-982 en de sim-backups op OVH en sub6 opgeruimd.
|
||||
**Naar 1.0.3 (besluit 2026-11-06):** lege mappen komen niet terug (3.982 op nc01, 2.076 daarvan in
|
||||
`files`, verklaart vrijwel zeker de 2.370 "removed" bij `files:scan`). De backup legt alleen
|
||||
bestanden vast (`walk()`), dus oudere snapshots hebben ze niet. Voor een DR acceptabel, wel
|
||||
fixen; issue #27. Daarna kleine volledige simulaties; een DR van 1 TB is hiervoor niet nodig.
|
||||
**Salvage vóór de release getest (besluit 2026-11-06):** nieuwe modus `salvage-files` in
|
||||
`full-restore-roundtrip.sh`, ook in CI: één pack beschadigd en één weg. Eerst stopt de restore,
|
||||
daarna `--salvage --force` volgens de gids. Vond één fout: bij een ontbrekend bestand wees de
|
||||
melding niet naar `--salvage` (opgelost).
|
||||
Stand 2026-11-04 (`polish`, gepusht): cron-waarschuwing + tick vanaf de pagina +
|
||||
voortgang van de db-fase; `--salvage` voor `backup:restore` en `backup:restore:full`; S3
|
||||
vooruitlezen (8 ranges tegelijk, met `SaveAs` naar de spool); SMB in leesacties van 8 MiB
|
||||
(gemeten met netem, 130 ms RTT: 4,5 → 18 MiB/s per leesactie; een volledige restore over SMB
|
||||
met vertraging is nog niet gemeten); Bin-tabblad; schatting `target:add`; nieuwe screenshots
|
||||
(1–3 vervangen, 4-bin erbij; in `info.xml` bij de release naar tag v1.0.2 laten wijzen en
|
||||
4-bin toevoegen). Getest: unit (208), psalm, integratie NC35 sqlite/pgsql, S3/SFTP/FTP/SMB,
|
||||
volledige restore met `--salvage`. **1.0.2 uitgebracht 2026-11-06** (tag `v1.0.2`, PR #28).
|
||||
- **Naar 1.0.3, uit de reviews van 1.0.2 (achteraf gedraaid, 2026-11-06):**
|
||||
1. Lege mappen (#27).
|
||||
2. Een volle schijf onder `--salvage` telt als beschadigde backup en de restore gaat door; zonder
|
||||
`--salvage` wijst de melding er ten onrechte naar. `fwrite` controleren, schrijffouten stoppen de
|
||||
restore (`Repository::restoreEntries`; ook `RestoreService::writeFile` nakijken). Beide reviews.
|
||||
3. Hervatten: controle op de doelmap vóór het overslaan, en tijdelijke bestanden ook opruimen in
|
||||
een map waar alles al klopt.
|
||||
4. Alleen `.ngb-restore-spool-<hex>` opruimen.
|
||||
5. Salvage-test: tijdelijke bestanden recursief zoeken; `admin` en `frtest` apart controleren.
|
||||
6. Nog nakijken: lease verversen bij overgeslagen bestanden; fout ná de database-commit
|
||||
(`restoreOwnAppConfig`) en tabellen die na een rollback blijven staan; backups zonder cron
|
||||
via de pagina (eerlijke beurten, stappen die de webtimeout halen); Bin verversen en fouten
|
||||
tonen; SMB-uitzondering bij het openen van een pack onder `--salvage`.
|
||||
Niet doen: bij hervatten de inhoud vergelijken (grootte en mtime is ook het criterium van de backup).
|
||||
**Stand op branch `1.0.3` (2026-11-06):** punten 1-6 gebouwd. Lege mappen staan in het metarecord
|
||||
van de snapshot, niet tussen de bestanden: 1.0.2 en ouder lezen zo'n snapshot gewoon en slaan alleen
|
||||
de lege mappen over. Geen webtimeout-wijziging zonder cron: alleen in de code nagelopen, niet getest
|
||||
(Nextcloud zet de PHP-tijdslimiet ruim, een gestopte stap wordt overgedaan); wel eerlijke beurten.
|
||||
Bin leest in delen met teller en balk (besluit 2026-11-06, ook met oog op meerdere backups per
|
||||
dag). Getest: unit (volle schijf op echte tmpfs), integratie NC35 sqlite/pgsql/mysql, sftp, salvage,
|
||||
salvage-files; Bin in een echte browser met vertraagde verzoeken (1 van 7 t/m 7). Nog: reviews
|
||||
vóór de PR, README-regels bij de versiecommit.
|
||||
**Simulaties (2026-11-06, NC 34.0.4, 2 GB, lokale SFTP, `~/dev-local/ngb-sim/sim.sh`):** normaal
|
||||
(pgsql), stop en hervat (pgsql → MariaDB, restore met kill -9 gestopt), volle schijf met `--salvage`:
|
||||
alle drie geslaagd, alle 3.101 gebruikersbestanden en 7 lege mappen gelijk, share, agenda en WebDAV
|
||||
werken. Vond één fout, opgelost: na een kill bleven vergrendelingen tot 2 uur staan, waardoor de
|
||||
backup van stap 11 werd geweigerd. `files:scan` "Updated" zijn in de gemeten run alleen mappen
|
||||
(mtime van mappen wordt niet teruggezet, dus nieuwe mtime en etag; bestanden ongemoeid). "Removed":
|
||||
o.a. `cache` van een gebruiker, die de backup uitsluit. Waarschijnlijk verklaart dit ook de 50.080
|
||||
"updated" op nc01, niet op nc01 zelf vastgesteld.
|
||||
**Naar 1.1.0 of 1.2.0 (besluit 2026-11-06):** alle mappen krijgen bij een restore hun oorspronkelijke
|
||||
tijd terug. Hoort zo, vooral bij oudere verzamelingen: de maptijd wordt gebruikt om in te schatten
|
||||
wanneer een map is aangemaakt. De backup moet dan de tijd van alle mappen vastleggen en de restore
|
||||
die aan het eind terugzetten; werkt pas vanaf de eerste backup met die versie. Op de roadmap onder 1.1.0.
|
||||
**1.0.3 uitgebracht (2026-11-06)**, ook op nc01 geïnstalleerd.
|
||||
- **Branches (besluit 2026-11-06):** werk voor een versie op een branch met die naam (`1.0.3`);
|
||||
`polish` alleen voor kleine cosmetische zaken, per geval bekijken of en wanneer die gemerged
|
||||
worden. SMB staat uit op sub5 en sub6; sub6 is het testaccount, eerst aanzetten voor een SMB-test.
|
||||
Na 1.0.3 gaat het werk naar `1.1.0`. Een spoedfix die niet kan wachten gaat op `1.0.4` vanaf `main`,
|
||||
krijgt een eigen merge en release; daarna `main` in `1.1.0` mergen, zodat daar met de fix getest wordt.
|
||||
- **1.0.1** (uitgebracht 2026-11-01): alleen de kleine punten (besluit 2026-10-30): 4, 6–13. Punten 1, 2 en 5 gaan naar
|
||||
1.0.2 (README-roadmap aangepast); punt 3 is een test, ingepland voor de week van 2 november,
|
||||
niet aangekondigd en los van de release. Voor 1.0.2 ook: nieuwe screenshots (het labelveld bij
|
||||
"Back up now" en het tabblad Docs staan er nog niet op; besluit 2026-11-01), en de prullenbak
|
||||
van herstelpunten naar een eigen tabblad "Bin" naast "Backup and restore", "Settings" en "Docs":
|
||||
onder de lijst met herstelpunten vervuilt hij het scherm (vooral met wat het bewaarbeleid erin
|
||||
zet; besluit 2026-11-01). Nieuwe screenshots daarna, zodat de Bin er ook op staat.
|
||||
1. **Naar 1.0.2.** (Naar rc4: herstelsnelheid over SFTP.) S3 leest al 16 tegelijk bij index en tree, maar een
|
||||
restore leest de packs nog één voor één: ook vooruitlezen via de S3-client.
|
||||
2. **Naar 1.0.2.** Achtergrondjobs zonder systeem-cron (issue #13; besluit 2026-10-19). De PR die dit
|
||||
oplost zegt "Fixes #13" (bevestigd: #13 was de cron; besluit 2026-10-21):
|
||||
a. **Waarschuwen** in de beheerpagina en `backup:status`: modus uit `core backgroundjobs_mode`
|
||||
(`ajax`/`webcron`/`cron`) en de laatste run uit `core lastcron`. Niet op `cron`, of de
|
||||
laatste run te lang geleden: melding met de cron-regel erbij ("Backups only move while
|
||||
someone has Nextcloud open"), en "waiting for background jobs" als een lopende backup lang
|
||||
geen stap zette.
|
||||
b. **De pagina laat de backup lopen** zolang ze openstaat: in `ajax`/`webcron` voert de
|
||||
statusaanroep die de pagina toch al herhaalt ook een korte, begrensde stap uit (zoals de
|
||||
korte stappen die AJAX-cron nu al krijgt). "Back up now" werkt dan ook zonder cron zolang
|
||||
het tabblad openstaat; de melding zegt dat geplande backups echte cron nodig hebben.
|
||||
De melding zegt (zonder werkende systeem-cron): backups hebben systeem-cron nodig, ook de
|
||||
gewone nachtelijke; zo stel je cron in (link naar de Nextcloud-documentatie over
|
||||
achtergrondjobs voor de versie van de server, nu
|
||||
https://docs.nextcloud.com/server/34/admin_manual/configuration_server/background_jobs_configuration.html,
|
||||
versienummer uit de server, niet vast); zolang er geen cron is, loopt een backup die je
|
||||
met de knop start alleen door terwijl de NG Backup-pagina openstaat.
|
||||
c. **Voortgang in de databasefase** (issue #13): nu staat er alleen "running (db)", wat er
|
||||
hetzelfde uitziet als vastzitten. Toon "running (db: 47/183 tables)" zoals de bestandsfase
|
||||
een percentage toont, en "waiting for background jobs (last ran 12 min ago)" als een run
|
||||
een tijd geen stap zette, ook bij een cron die is blijven staan.
|
||||
Niet doen: de cron-modus vanuit de app omzetten (zonder echte cron-regel stopt alles, en
|
||||
`cron.php` zet de modus zelf op `cron`), en geen `exec php occ backup:run &` vanuit de knop
|
||||
(vaak uitgeschakeld, niet op hosted, buiten Nextcloud om lastig te volgen en te stoppen).
|
||||
3. Volledige DR-test met een backup die al de tabelstructuur bevat (gemaakt met rc2 of later),
|
||||
zonder `--skip-missing-tables`: dat pad is op 1 TB nog niet getest.
|
||||
4. **Label bij "Back up now"** (besluit 2026-10-19): een optioneel tekstveld naast de knop; de
|
||||
tekst gaat als label van de snapshot mee (nu kan dat alleen met `occ backup:run -l`; de knop
|
||||
stuurt een leeg label, geplande runs krijgen "scheduled"). Raakt alleen de UI en een
|
||||
bestaande parameter. Ten hoogste 100 tekens — **klaar** (2026-10-30).
|
||||
5. **`--salvage`** voor `backup:restore` en `backup:restore:full` (besluit 2026-10-22): terughalen
|
||||
wat kan, een bestand dat niet te lezen of te ontsleutelen is overslaan en aan het eind
|
||||
opsommen, in plaats van stoppen. **Naar 1.0.2** (besluit 2026-10-30).
|
||||
6. **Tijdelijke bestanden van een afgebroken restore opruimen** (gevonden in de SMB-DR van
|
||||
rc6, 2026-10-24): de rc5-poging die stopte liet `VID_0009.mp4.ngb-restore-<id>` staan; de
|
||||
geslaagde restore daarna haalde het niet weg, en `files:scan` maakte het zichtbaar in Files.
|
||||
Een restore ruimt `*.ngb-restore-*` in de mappen die hij terugzet op (alleen bestanden met
|
||||
precies dat achtervoegsel en 6 of 12 hextekens) — **klaar** (2026-10-30).
|
||||
7. **Eén bestand terugzetten met `backup:restore`**: een pad naar een bestand zet 0 bestanden
|
||||
terug (er komt een `/` achter); nu werkt alleen een map. Gold alleen voor `--to-directory`;
|
||||
naar de bestanden van een gebruiker werkte een los bestand al — **klaar** (2026-10-30).
|
||||
8. **Uitleg (i) bij de prullenbak van herstelpunten** (besluit 2026-10-25): bij "removed by
|
||||
retention policy" een infoknop met wat het betekent: het bewaarbeleid houdt voor oudere dagen
|
||||
één herstelpunt per dag (het laatste), weken en maanden; de rest gaat naar de prullenbak en
|
||||
wordt pas na de ingestelde wachttijd (`delete_delay_days`, standaard 30) echt verwijderd, en is
|
||||
tot die datum terug te zetten. Aanleiding: de beheerder zag "19-10 13:12 – after rc4 upgrade
|
||||
removed by retention policy" en dacht dat er iets was weggegooid. `occ backup:trash` zegt
|
||||
hetzelfde onder de lijst — **klaar** (2026-10-30).
|
||||
9. **Tijdzone in `occ backup:status`**: de tabel toont de servertijd (UTC), de webinterface de
|
||||
tijd van de beheerder; dat leest als een verkeerde starttijd. Zet de zone in de kolomkop
|
||||
("started (UTC)") of toon de tijden in de zone van het schema. Gedaan: beide, en ook in
|
||||
`backup:list`, `backup:trash` en `backup:browse` — **klaar** (2026-10-30).
|
||||
10. **User-restore op gebruikersnaam** (issue #22, 2026-10-27): `backup:user:restore <locatie>
|
||||
New_User` gaf alleen "is not a recorded user export"; het argument moet het manifestpad
|
||||
zijn, dat alleen `backup:user:backup` eenmalig toonde. Nu: `backup:user:list` toont de
|
||||
exports, een kale uid werkt bij precies één export, anders een lijst om uit te kiezen —
|
||||
**klaar** (2026-10-27).
|
||||
11. **Tabblad "Docs" in de beheerpagina** (besluit 2026-10-27): een derde knop naast "Backup and
|
||||
restore" en "Settings", met links naar de HOWTO (`docs/HOWTO.md`), de DR-gids
|
||||
(`docs/DISASTER-RECOVERY.md`) en de roadmap (README, `#roadmap`) op GitHub, elk met een
|
||||
regel wat erin staat. Nu zijn ze alleen via de README te vinden. Zichtbaar ook zonder
|
||||
sleutel, zodat de gids vindbaar is vóór de eerste installatie-stap. "NG Backup is in beta"
|
||||
gaat uit de feedbackregel — **klaar** (2026-10-27).
|
||||
12. **SFTP-sleutel zonder regeleinden** (issue #24, 2026-10-28): het veld "Private key" op de
|
||||
beheerpagina is één regel, dus een geplakte sleutel verliest zijn regeleinden. NC34
|
||||
(phpseclib 2) las dat nog, NC35 (phpseclib 3) weigert een OpenSSH-sleutel dan met "Unable
|
||||
to read key". De regeleinden worden nu teruggezet vlak voor gebruik — **klaar** (2026-10-28).
|
||||
13. **"database is locked" bij het begin van `backup:restore:full` op SQLite** (CI, 2026-10-26
|
||||
bij #22 en 2026-10-29 op `main` na de merge van 1.0.0): de restore stopt meteen na het
|
||||
aanzetten van de onderhoudsmodus. Niet altijd, dus een tweede proces dat op hetzelfde moment
|
||||
de database gebruikt (cron, een webverzoek of de opstart van de container). Kan ook bij een
|
||||
echte restore naar SQLite gebeuren. Oorzaak (2026-10-29): SQLite (WAL) begint een transactie
|
||||
als lezer; schrijft een ander proces tussen ons eerste lezen en ons eerste schrijven, dan
|
||||
faalt dat schrijven direct (geen wachttijd helpt). In CI was dat de installer, die
|
||||
"installed" zet en daarna nog seconden doorschrijft. Nu neemt de restore eerst het
|
||||
schrijfslot en wacht hij op de ander; de SQLite-test draait met een tweede schrijver
|
||||
ernaast — **klaar** (2026-10-29).
|
||||
- **DR-tests over databasegrenzen** (besluit 2026-10-20; productie nc01 draait PostgreSQL): na
|
||||
de MariaDB→PostgreSQL-fout (tellers, rc5) elke overstap in de integratietest (`to-pgsql`,
|
||||
`to-mysql`, `to-sqlite`, ook in CI). Volle DR-tests met echte data: vóór 1.0.0 de S3-simulatie
|
||||
(MariaDB-bron, 138 GB) nogmaals naar een MariaDB-doel en de SMB-simulatie naar een SQLite-doel;
|
||||
**na 1.0.0** de 1 TB van nc01 (PostgreSQL) naar een MariaDB-doel.
|
||||
- **Polish:** `target:add` toonde bij het eerste indexbestand al "about 642 min left" (S3,
|
||||
4.568 bestanden, daarna binnen een minuut klaar): een schatting pas na ~5% of na meer dan
|
||||
een paar bestanden.
|
||||
- **1.1.0** (geen harde datum: een week was een voorzet, langer mag; de releasefrequentie mag omlaag,
|
||||
besluit 2026-11-06):
|
||||
**Betere melding voor namen die een snapshot niet kan bevatten** (backslash, stuurteken, geen UTF-8):
|
||||
een eigen melding in plaats van "unreadable" met het advies over rechten. Zegt dat deze bestanden of
|
||||
mappen in de backup ontbreken, geeft voorbeelden van de paden en vraagt ze te hernoemen; de volgende
|
||||
backup neemt ze mee. Alleen als Nextcloud-melding: de app mailt niet zelf, of meldingen gemaild worden
|
||||
is de keuze van de beheerder (besluit 2026-11-06).
|
||||
**Stand op branch `1.1.0` (2026-11-07):** klaar: betere melding, punten 2, 4, 5, 6 en 8, en de maptijden
|
||||
(`files:scan` na een volledige restore: 0 "Updated" in de simulatie; opslag gepakt in het metarecord).
|
||||
Punt 2 reserveert de jobs in `oc_jobs` met één tijdstempel, alleen wat Nextcloud op dat moment zou kunnen
|
||||
starten; 6 uur aan, dan 1 uur vrij; bij een crash geeft Nextcloud de reservering na 12 uur zelf vrij;
|
||||
de logregels staan op niveau info, dus standaard niet zichtbaar. Pin en notitie staan
|
||||
in de catalogus, met een kopie in de database voor de lijsten. Een moment van vóór het instellen van
|
||||
een schema start niets. Geen schema: geen melding "geen recente backup", wel een melding aan alle
|
||||
beheerders bij het uitzetten en een herinnering, wekelijks zolang een andere locatie een schema heeft,
|
||||
anders dagelijks (besluit 2026-11-07). Nog niet getest: de update vanaf 1.0.3, de pagina in een browser.
|
||||
1. `backup:key:import` zonder kit: de sleutel-slots van de locatie (`config`) lezen, met de kit
|
||||
als reserve als `config` ontbreekt of niet opent. Vergt een andere volgorde van
|
||||
`target:add` en sleutelimport (het gevoeligste pad), daarom niet in rc1. Daarna de
|
||||
DR-test op PVE02 herhalen zonder kit.
|
||||
**Gebouwd (2026-11-07):** `backup:target:add` op een server zonder sleutel kijkt eerst of de
|
||||
locatie een repository heeft, vraagt dan een wachtwoordzin (of `--passphrase-file`), opent de
|
||||
slots uit `config`, controleert de sleutel tegen de `check` en slaat hem pas dan op; geen
|
||||
kitbevestiging, dus backups wachten op een nieuwe kit. Nieuw bestand `settings` op de locatie,
|
||||
versleuteld met de hoofdsleutel en eenmalig geschreven (bij `target:add` en bij het begin van
|
||||
een backup): de verwijdertermijn, anders de standaard 7 dagen. Getest: unit-tests en
|
||||
`full-restore-roundtrip.sh 35-apache pgsql no-kit`. Nog open: de DR-test op PVE02 zonder kit.
|
||||
2. Instelling "zware achtergrondjobs pauzeren tijdens een backup" (standaard aan, uit te zetten):
|
||||
Memories, files_antivirus BackgroundScanner, Recognize, Preview Generator, fulltextsearch.
|
||||
Via de reservering/last_run van die jobs (geen apps uitzetten), crashbestendig, maximaal
|
||||
@@ -395,6 +606,97 @@ in README "Known limitations":
|
||||
3. Snelheid: lezen en uploaden laten overlappen, meerdere bestanden tegelijk lezen, eventueel
|
||||
een eigen worker (`occ background-job:worker`). Gemeten op nc01: 55% wachten op de schijf,
|
||||
33% op de SMB-upload, ~7% CPU. Vergt extra PHP-processen; botst mogelijk met uitgangspunt 1.
|
||||
**Gemeten 2026-11-07 op nc01 (alleen lezen):** de schijf (sdb, 2 TB, ROTA=1) haalt met `dd`
|
||||
21 MiB/s met 1 stroom, 37 met 4, 55 met 8. PHP las 8 KiB per verzoek: 5,2 MiB/s. Met
|
||||
`stream_set_read_buffer(0)` en blokken van 4 MiB: 26,8 MiB/s. Hulpprocessen die vooruitlezen
|
||||
gaven daarbovenop niets (20 MiB/s, ze concurreren). Gebouwd: grote leesverzoeken (commit
|
||||
"Read files ... 4 MiB"). Niet gebouwd: meerdere uploads tegelijk; SMB bundelt al per 1 MB
|
||||
(icewind/smb), dus winst over een lange lijn vraagt meerdere packs tegelijk.
|
||||
**Besluit 2026-11-08: meerdere uploads tegelijk in 1.2.0**, met een instelling (standaard 2)
|
||||
zodat de beheerder het afstemt op zijn server en de omvang van zijn backup.
|
||||
4. **Schema per locatie** (besluit 2026-10-19): in de instellingen van een locatie "uit",
|
||||
"dagelijks om HH:MM" of "elke N uur (vanaf HH:MM)". Nu is er één `schedule_time` voor alle
|
||||
locaties, één keer per dag. Bij de update wordt de huidige tijd het schema van elke
|
||||
bestaande locatie (voor bestaande gebruikers verandert er niets).
|
||||
5. **Locaties na elkaar laten starten** in plaats van tegelijk: twee runs die dezelfde schijven
|
||||
lezen zijn samen niet sneller (gezien op nc01); wachtende runs zichtbaar als "queued".
|
||||
7. **SMB sneller** (gemeten 2026-10-21): lokaal netwerk zonder vertraging (Samba-container op de
|
||||
laptop, 1,8 GB) back-up 24,9 MB/s, restore 27,6 MiB/s; vanaf Canada naar de Storage Box (130 ms)
|
||||
2,6 MiB/s, omdat SMB per blok op het antwoord wacht. Grotere schrijfblokken, en vooruitlezen
|
||||
bij een restore zoals SFTP (IReadsAhead) of meerdere packs tegelijk. Een NAS met 1 Gbit haalt
|
||||
~110 MB/s: daar is winst.
|
||||
Ook de index: `target:add` over SMB vanaf Canada haalt ~1 indexbestand per seconde (4.927
|
||||
bestanden ~85 min), want SMB krijgt geen `getMany()` met verzoeken tegelijk zoals S3 en SFTP.
|
||||
6. **Notitie achteraf bij een restorepunt**, te wijzigen in de lijst. Het label in de snapshot
|
||||
is versleuteld en ligt vast; de notitie apart op de locatie bewaren (bijv. in de catalogus),
|
||||
zodat een nieuwe server na een DR hem ook ziet.
|
||||
8. **Herstelpunt vastpinnen** (besluit 2026-10-25): een herstelpunt dat het bewaarbeleid
|
||||
nooit opruimt (bijv. "vóór de upgrade naar NC 35"), in de lijst en met
|
||||
`occ backup:pin <snapshot>` / `--unpin`. Op de locatie bewaren (catalogus), zodat de
|
||||
opschoning het ziet, ook na een DR; vastgepinde punten tellen niet mee in last/daily/weekly.
|
||||
9. **Feedbackvraag weghalen in 1.2.0** (besluit 2026-11-07, niet in 1.1.0): de regel "We would like
|
||||
to hear how NG Backup works for you ... Share your experience" in de beheerpagina blijft tot
|
||||
dan staan. Er komt weinig feedback bij zo'n 150 downloads per versie; geen bericht is goed
|
||||
bericht. Het formulier (`experience.yml`) in de issues blijft.
|
||||
- **Bevindingen DR-test op PVE02 (2026-10-17, snapshot van nc01, 1 TB, AIO → gewone NC 34 met
|
||||
PostgreSQL 18)** — voor **1.0.0-rc.2**:
|
||||
1. Tabellen van apps die bij de backup **niet aanstonden** (restanten: Deck, ONLYOFFICE,
|
||||
AppAPI, een archief-app) blokkeren de restore met één tabelnaam per poging. Standaard
|
||||
overslaan met één waarschuwing per app; optie om ze toch terug te zetten.
|
||||
2. Alle ontbrekende apps en tabellen **vooraf in één lijst**, per app benoemd (nu één per poging,
|
||||
ook voor uitgeschakelde standaard-apps zoals `admin_audit`).
|
||||
3. `--skip-missing-apps`; `nextcloud-aio` standaard overslaan (bestaat buiten AIO niet), zodat
|
||||
AIO → gewone Nextcloud werkt.
|
||||
4. `--install-apps`: ontbrekende apps uit de app store installeren.
|
||||
5. Apps buiten de app store (eigen builds, `side_menu`, `nextcloud-aio`): documenteren; later
|
||||
eventueel `custom_apps` meenemen in de backup.
|
||||
6. Tabellen van een app die wél aanstond maar die hun tabel later zelf maakt (Memories:
|
||||
`memories_planet_geometry`) netjes afhandelen.
|
||||
7. `backup:target:add` zonder sleutel verwijst naar `key:init`; als er al een repository staat,
|
||||
moet het naar `key:import <kit>` wijzen.
|
||||
8. `backup:target:add` toont geen voortgang bij het ophalen van de index (27.582 bestanden).
|
||||
9. `backup:target:remove` laat de index-cache van die locatie in de database staan; opruimen,
|
||||
ook bestaande wezen.
|
||||
10. Snapshot-meta uitbreiden: ook geïnstalleerde maar uitgeschakelde apps, met per app of hij
|
||||
uit de app store komt; `occ backup:apps <locatie> <snapshot>` toont de lijst leesbaar
|
||||
(geen platte tekst op de locatie: dat lekt welke apps/versies er draaien).
|
||||
11. **Tabelstructuur meedumpen** (kolommen, typen, indexen, primary key), zodat de restore een
|
||||
ontbrekende tabel zelf aanmaakt met de gegevens erin. Lost restanten van oude apps op zonder
|
||||
ze over te slaan; een later opnieuw geïnstalleerde app vindt zijn oude gegevens terug.
|
||||
12. **Foreign keys**: de restore zet tabellen op alfabet terug; `oc_mail_accounts` verwijst naar
|
||||
`oc_mail_mailboxes` en PostgreSQL weigert (fk_61a7e87f59a77cdc). Iedereen met Mail loopt hier
|
||||
vast. Oplossing: in de juiste volgorde terugzetten (afhankelijkheden eerst), of de controle
|
||||
tijdelijk uitzetten en na afloop terugzetten en controleren. In de test: 15 FK's (alle van
|
||||
Mail) tijdelijk verwijderd en na de restore teruggezet: alle 15 geldig op de teruggezette
|
||||
data, dus "uitzetten, terugzetten, achteraf controleren" werkt.
|
||||
13. **SFTP-restore over een lange lijn is traag**: 0,75 MiB/s vanaf Hetzner naar Canada (RTT 122 ms),
|
||||
terwijl OpenSSH `sftp` 16,6 MiB/s haalt. De SFTP-stream van files_external (phpseclib) wacht na
|
||||
elk klein blok op antwoord. Oplossing: pack-ranges bij SFTP met phpseclib's eigen `get()`
|
||||
(veel verzoeken tegelijk) lezen in plaats van via de stream, zoals bij S3. 1 TB duurt nu ~2 weken.
|
||||
Na meten: de grootste oorzaak was het netwerk (cubic, zie 14): met BBR gaat de restore van 0,75
|
||||
naar 2,3 MiB/s, een hele pack lezen 4-5 MiB/s. Meer verzoeken in één stream (DEPTH 16/32) gaf geen
|
||||
duidelijke winst. Verder komen vraagt meerdere verbindingen tegelijk: naar **1.1.0**.
|
||||
14. **Uploaden over een lange lijn**: backup naar OVH S3 Warschau vanaf Canada (RTT 114 ms) haalt
|
||||
~1 MiB/s met één pack tegelijk; `aws s3 cp` (multipart, parallel) 6 MiB/s. Meerdere packs tegelijk
|
||||
uploaden of S3 multipart. Minder dringend dan 13 (restore).
|
||||
Oorzaak netwerk (met Sysop): pakketverlies trans-Atlantisch + cubic in de VM; met BBR gaat één
|
||||
PUT van 2,4 naar 7,5 MiB/s, ng_backup van ~1 naar ~1,6 MiB/s (rest = serieel lezen/uploaden).
|
||||
Tip over BBR in de README gezet. Let op: PVE02 staat in Canada; een restore in de EU gaat sneller.
|
||||
15. DR-document: in het Docker-voorbeeld "je containernaam" (`docker ps`) i.p.v. `nextcloud`.
|
||||
16. Grote restore (VM 980, snapshot aa7cce8d van de Storage Box) stopte na 22 uur met
|
||||
`Not found: packs/26/2642ef39…`. Oorzaak: de wekelijkse opruiming van nc01 (zo 18-10) heeft
|
||||
dat pakket tijdens de restore omgepakt (blob staat nu in `389567b7…`); de herstelserver had de
|
||||
index van vóór de opruiming. Productie heel: `backup:verify` op nc01 OK (150.639 bestanden,
|
||||
316.830 blobs). Opgelost in rc2: bij een mislukte pakketlezing de index herladen en de blob
|
||||
lezen waar hij nu staat (test `testRestoreFollowsPacksRepackedByAnotherServer`).
|
||||
**Kleine DR-test met rc2 (2026-10-18), volautomatisch geslaagd:** bron VM 981 (NC 34, PostgreSQL,
|
||||
51 GB, Mail met gesynct account, Deck uitgeschakeld, Calendar/Contacts, admin_audit) → OVH S3 →
|
||||
verse VM 982 met alleen de DR-gids en `--install-apps`: apps vanzelf geïnstalleerd, 10 Deck-tabellen
|
||||
aangemaakt, 15 FK's in orde, 23.519/23.519 checksums gelijk, agenda/contact/mail/bord terug.
|
||||
Restore 1 u 47 vanuit Warschau naar Canada (met BBR). Lokaal slaagt de test ook op SQLite en MariaDB.
|
||||
**Daarna verplicht:** een tweede volledige restore op een verse VM, alleen met de HOWTO en de
|
||||
nieuwe opties, zonder handwerk. Pas als die in één keer slaagt: 1.0.0.
|
||||
|
||||
- **Later (geen versie):** optie om de sleutel niet op de locatie op te slaan (kit is dan de enige
|
||||
kopie; uitdrukkelijke waarschuwing). Primaire object storage en External-storage-koppelingen
|
||||
meenemen via de opslaglaag (§3.4; nu niet ondersteund, zie README "Known limitations").
|
||||
|
||||
@@ -7,11 +7,17 @@ installations and hosted Nextcloud. (Choosing SMB as a storage backend is the on
|
||||
see "Storage backend experiences" below — that requirement comes from files_external, not from
|
||||
ng_backup, and applies to any Nextcloud app that writes to an SMB share.)
|
||||
|
||||
**Status: release candidate for 1.0.0 (1.0.0-rc.1, app version 0.9.7).** Tested thoroughly, usable
|
||||
alongside your current backup, but not yet a drop-in replacement for it. See "Known limitations" below, `PLAN.md` for the design and
|
||||
`docs/HOWTO.md` for a step-by-step guide from setup to a tested restore.
|
||||
**Status: 1.1.0, stable.** Disaster recovery has been tested on real installations (see "Tested
|
||||
disaster recoveries" below). See "Known limitations" below, `PLAN.md` for the design and
|
||||
`docs/HOWTO.md` for a step-by-step guide from setup to a tested restore. Server lost? See
|
||||
`docs/DISASTER-RECOVERY.md`.
|
||||
|
||||
License: AGPL-3.0-or-later.
|
||||
If you like NG Backup, don't forget to give the app a star in the
|
||||
[app store](https://apps.nextcloud.com/apps/ng_backup) and this repository a star on GitHub!
|
||||
|
||||
License: AGPL-3.0-or-later. Developed and sponsored by [Ensembia](https://www.ensembia.com).
|
||||
Need help restoring a backup after a crash? Ensembia offers paid restore assistance:
|
||||
[www.ensembia.com](https://www.ensembia.com).
|
||||
|
||||
## How it works
|
||||
|
||||
@@ -30,6 +36,27 @@ License: AGPL-3.0-or-later.
|
||||
- Restores go through Nextcloud, so overwritten files keep their old content as a version and
|
||||
removed files go to the trash.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Background jobs on system cron.** NG Backup runs backups in steps from Nextcloud's
|
||||
background job, so a backup only moves forward when that job runs. Choosing "Cron" under
|
||||
**Administration settings → Basic settings → Background jobs** is not enough on its own: it
|
||||
tells Nextcloud that the system will call it, so you also need a cron line that does. Without
|
||||
it scheduled backups do not run; the admin page and `occ backup:status` warn about it, and a
|
||||
backup started with "Back up now" only moves on while the NG Backup page stays open.
|
||||
`occ backup:run` in a terminal works without cron (it runs in the foreground).
|
||||
- Regular server: `crontab -u www-data -e` and add
|
||||
`*/5 * * * * php -f /var/www/nextcloud/cron.php` (your Nextcloud path).
|
||||
- Docker (the official `nextcloud` image): on the host, `crontab -e` as root and add
|
||||
`*/5 * * * * docker exec -u www-data <container> php -f /var/www/html/cron.php`, or add a
|
||||
service with the same image and `entrypoint: /cron.sh` to your `docker-compose.yml`.
|
||||
- Nextcloud All-in-One already runs cron.
|
||||
|
||||
After a few minutes, the Background jobs section shows that the last job ran just now. See the
|
||||
[Nextcloud documentation on background jobs](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/background_jobs_configuration.html).
|
||||
- A storage location: S3 or compatible, SFTP, WebDAV, SMB (needs the PHP `smbclient` extension,
|
||||
see below), FTP or a local path.
|
||||
|
||||
## Quick start
|
||||
|
||||
```
|
||||
@@ -60,8 +87,9 @@ All backups of an installation are encrypted with one master key.
|
||||
- The **recovery kit** is your own copy of that encrypted key. Still keep it off the server and
|
||||
off the location: an attacker who gets write access to the location can damage or delete its
|
||||
`config` file on purpose, and without the kit that makes every backup there unreadable.
|
||||
- For a **restore on a new server** (disaster recovery) you currently need the **recovery kit**
|
||||
and **one passphrase**.
|
||||
- For a **restore on a new server** (disaster recovery) **one passphrase** is enough: the new
|
||||
server takes the key from the location. With the **recovery kit** you can restore even when the
|
||||
location's `config` file is damaged or gone.
|
||||
- Planned: an option to not store the key on the location at all. Then the kit is the only copy,
|
||||
and losing it means losing every backup; that option will come with an explicit warning.
|
||||
- Up to **three passphrases** ("slots") can open the key, so one person leaving or dying does not
|
||||
@@ -123,10 +151,34 @@ What you should do on the location (strongly recommended):
|
||||
- **Files and folders** of a user: in the web interface or `occ backup:restore`, into a new folder
|
||||
("Restored <date>"), merged into the original place, or replacing it.
|
||||
- **Raw restore** of any path (for example `config/`) to a local directory: `occ backup:restore ... --to-directory=DIR`.
|
||||
- **Whole server** (disaster recovery onto a fresh installation): `occ backup:restore:full`, see the how-to.
|
||||
- **Whole server** (disaster recovery onto a fresh installation): `occ backup:restore:full`, see
|
||||
[docs/DISASTER-RECOVERY.md](docs/DISASTER-RECOVERY.md). It runs in maintenance mode, so that
|
||||
nobody and no background job writes into the instance while it is being replaced.
|
||||
- **One user** (account, settings, files, calendars, contacts via user_migration): `occ backup:user:backup` and
|
||||
`occ backup:user:restore`, replacing the account or restoring it as `<uid>-bak`. See the how-to.
|
||||
|
||||
**Without shell access** (a hosted Nextcloud where you cannot run `occ`): files and folders can be
|
||||
restored in the web interface. Restoring a whole user or the whole server needs `occ`, so on such
|
||||
hosting it is something your provider (or whoever has shell access) runs. A restore mode that
|
||||
works without maintenance mode, and so from the web interface, is on the roadmap.
|
||||
|
||||
## Tested disaster recoveries
|
||||
|
||||
Full restores onto a fresh server, following only [the guide](docs/DISASTER-RECOVERY.md):
|
||||
|
||||
| Backup | Location | Database | Result |
|
||||
|---|---|---|---|
|
||||
| Production, about 1 TB, 150,000 files (Nextcloud All-in-One) | Hetzner Storage Box (SFTP), Germany → Canada | PostgreSQL → PostgreSQL | 150,622 files restored in about 25 h (13-14 MiB/s), sample identical |
|
||||
| The same, 1.0.2 | Hetzner Storage Box (SMB), Germany → Canada | PostgreSQL → PostgreSQL | 150,755 files restored in about 29 h (7-13 MiB/s), going on after a lost connection; sample identical, empty folders not restored ([#27](https://github.com/dillardblom/ng_backup/issues/27)) |
|
||||
| 1.0.3, small simulations of 2 GB: as planned, with the restore killed halfway, and with a disk that runs full | Local SFTP server | PostgreSQL → PostgreSQL and MariaDB | all 3,101 user files and all empty folders identical; users, a share and a calendar back |
|
||||
| 1.1.0, the same three simulations | Local SFTP server | PostgreSQL → PostgreSQL and MariaDB | all 3,101 user files identical in content, size and time, all empty folders back; users, a share and a calendar back |
|
||||
| 138 GB simulation: 20 users, calendars, contacts, mail, shares, team folders with an ACL, versions, trash | OVH S3, Warsaw → Canada | MariaDB → PostgreSQL | all 15,772 files and every database count identical |
|
||||
| The same | OVH S3 | MariaDB → MariaDB | all 15,772 files and every database count identical |
|
||||
| The same | Hetzner Storage Box (SMB), Germany → Canada | MariaDB → SQLite | all 15,772 files and every database count identical (27.6 h, about 1.5 MB/s) |
|
||||
| 51 GB test instance | OVH S3 | PostgreSQL → PostgreSQL | 23,519/23,519 checksums identical |
|
||||
|
||||
The integration tests restore across every pair of PostgreSQL, MariaDB and SQLite.
|
||||
|
||||
## Storage backend experiences
|
||||
|
||||
Notes from actually running a backend, beyond what's in files_external's own documentation.
|
||||
@@ -172,16 +224,71 @@ Contributions welcome via PR — add what you ran into with your own provider.
|
||||
the backend being listed is not proof it will work here. Once the extension is present it works
|
||||
the same as any other backend; no ng_backup-specific config beyond host/share/user/password.
|
||||
|
||||
- **Backups over a long distance (high latency):** on Linux, the default TCP congestion control
|
||||
(cubic) slows a single connection down a lot on a long line with some packet loss. From Canada to
|
||||
an S3 bucket in Warsaw (114 ms), one upload went from 2.4 to 7.5 MiB/s after switching the
|
||||
server (or the VM/container host where TCP ends) to BBR:
|
||||
`sysctl -w net.core.default_qdisc=fq net.ipv4.tcp_congestion_control=bbr`
|
||||
(`modprobe tcp_bbr` first; make it permanent in `/etc/sysctl.d/`).
|
||||
A full restore over SFTP reads the packs ahead on a second connection, with many requests in
|
||||
flight. A server that handles parallel reads well (not a Hetzner Storage Box, which gets slower)
|
||||
can be read on more connections: `occ config:app:set ng_backup restore_sftp_connections
|
||||
--value=4` (at most 8; the server must allow that many plus one at once for the user).
|
||||
|
||||
- **OVH Object Storage (S3, Warsaw, 2026-10):** works with the `amazons3` backend (path style,
|
||||
region `waw`). A full backup of 1 TB from Germany ran at about 7 MiB/s, from Canada about
|
||||
6 MiB/s with BBR; a check of a 1 TB backup took 3.5 minutes (index files are fetched 16 at a
|
||||
time); a restore of 138 GB from Warsaw to Canada took about 6 hours. Turn on versioning and
|
||||
Object Lock on the bucket (see above); OVH charges no egress.
|
||||
|
||||
- **Hetzner Storage Box over SFTP, full restore:** about 13-14 MiB/s from Germany to Canada
|
||||
(122 ms) with the read-ahead of 1.0.0-rc.4; without it about 2.8 MiB/s. A Storage Box gets
|
||||
slower with several connections reading at once (4-6 MiB/s for 2-4 connections, it reads from
|
||||
disks), so one extra connection is the default. A Storage Box drops an SFTP connection that
|
||||
stayed quiet for a while without telling the client; NG Backup checks and replaces it.
|
||||
|
||||
- **SMB speed:** on a local network (Samba, no delay) a backup ran at about 25 MB/s and a
|
||||
restore at about 28 MiB/s. Over a long line SMB waits for an answer to each request: from
|
||||
Canada to a Storage Box in Germany (130 ms) about 2.6 MiB/s for a backup and 1.6 MiB/s for a
|
||||
restore, and connecting a location with an existing backup reads its index at about one file a
|
||||
second. SMB is meant for a local network; use SFTP or S3 for a location far away. When
|
||||
`pecl install smbclient` fails (pecl.php.net was unreachable for hours on 2026-10-21), the
|
||||
extension builds from its source: `libsmbclient-php` on GitHub, `phpize && ./configure && make
|
||||
&& make install`, then `docker-php-ext-enable smbclient`.
|
||||
|
||||
- **NFS is not a files_external backend at all.** Mount the NFS export on the server yourself
|
||||
(e.g. via `/etc/fstab` or a systemd `.mount` unit) and point ng_backup's `local` backend at the
|
||||
mount point; ng_backup never mounts or unmounts anything on its own, so an NFS mount dropping
|
||||
mid-run behaves like any other local-path failure (the run resumes). See PLAN.md 3.5 for why NFS
|
||||
can't be driven from PHP directly.
|
||||
|
||||
## Roadmap
|
||||
|
||||
What is planned, without dates. Ideas and votes are welcome in the
|
||||
[issues](https://github.com/dillardblom/ng_backup/issues); if something you need is listed here,
|
||||
a 👍 on its issue helps decide what comes first.
|
||||
|
||||
**1.2.0**
|
||||
- Faster backups over a long line: several uploads at the same time, with a setting
|
||||
(2 by default) to suit the server and the size of the backup.
|
||||
|
||||
**Later**
|
||||
- A full restore onto the existing installation, without first installing a fresh Nextcloud, in a
|
||||
restore mode of its own instead of maintenance mode, so that it can also run from the web
|
||||
interface (for hosted Nextcloud without shell access).
|
||||
- Users restoring their own files from the Files app.
|
||||
- Primary object storage and External Storage mounts (see Known limitations).
|
||||
- An option not to keep the backup key on the location (the recovery kit is then the only copy).
|
||||
|
||||
## Known limitations
|
||||
|
||||
On the backlog; none of these block the beta or 1.0.0, but know them before you rely on it:
|
||||
On the backlog; know them before you rely on NG Backup:
|
||||
|
||||
- **Empty folders only come back from backups made with 1.0.3 or newer.** Older restore points do
|
||||
not contain them.
|
||||
- **Folders with files in them only get their original modification time back from backups made with
|
||||
1.1.0 or newer.** From older restore points they show the time of the restore; the files keep theirs.
|
||||
A restore through Nextcloud into a folder that is already there leaves that folder's time alone.
|
||||
- **Only the data directory is backed up.** Object storage as primary storage (`objectstore` in
|
||||
`config.php`) is not supported: a backup refuses to start there. Folders mounted through the
|
||||
External storage app are not in the data directory and are **not included**, without a warning.
|
||||
@@ -202,8 +309,11 @@ On the backlog; none of these block the beta or 1.0.0, but know them before you
|
||||
- Location passwords passed with `-o password=...` are visible in the process list and shell
|
||||
history while the command runs.
|
||||
- Temporary files of a restore into a user's files end up in that user's trash.
|
||||
- Restoring the database does not order tables by foreign keys; apps with cascading foreign keys
|
||||
could lose rows (Nextcloud core does not use them).
|
||||
- **Restoring a whole user or the whole server needs `occ`** (shell access); see "Without shell
|
||||
access" above.
|
||||
- **`--install-apps` depends on the app store.** Nextcloud downloads the whole app list (about
|
||||
38 MB) and gives up after 120 seconds; on a slow day apps are not installed. The restore then
|
||||
lists them: install them afterwards with `occ app:install`, their data is restored already.
|
||||
- A pack whose index upload failed stays on the location until removed by hand.
|
||||
|
||||
## Commands
|
||||
@@ -213,11 +323,14 @@ On the backlog; none of these block the beta or 1.0.0, but know them before you
|
||||
| `backup:key:init`, `backup:key:kit`, `backup:key:confirm` | create the key, recovery kit, confirmation |
|
||||
| `backup:key:slots`, `backup:key:slot:add/replace/remove` | passphrase slots |
|
||||
| `backup:target:backends/add/edit/list/remove/append-only` | backup locations |
|
||||
| `backup:target:schedule` | when a location is backed up: off, daily, or every N hours |
|
||||
| `backup:run`, `backup:status`, `backup:list` | back up, status (runs of the last 7 days; `--all` for all), restore points |
|
||||
| `backup:cancel <run>` | stop a running backup |
|
||||
| `backup:browse`, `backup:restore` | browse and restore |
|
||||
| `backup:retention`, `backup:prune`, `backup:trash`, `backup:trash:restore` | retention, cleanup, trash |
|
||||
| `backup:pin`, `backup:note` | keep a restore point, a note you can change later |
|
||||
| `backup:pause-jobs` | whether heavy background jobs of other apps pause during a backup |
|
||||
| `backup:verify` | checksum audit of a restore point (`--deep` downloads and decrypts everything) |
|
||||
| `backup:restore:full` | whole-server restore onto a fresh installation (after `backup:key:import`) |
|
||||
| `backup:key:import` | import a recovery kit on a fresh installation |
|
||||
| `backup:user:backup`, `backup:user:restore` | one user via user_migration (asks before installing it) |
|
||||
| `backup:user:backup`, `backup:user:list`, `backup:user:restore` | one user via user_migration (asks before installing it); list the exports |
|
||||
|
||||
@@ -3,13 +3,49 @@
|
||||
xsi:noNamespaceSchemaLocation="https://apps.nextcloud.com/schema/apps/info.xsd">
|
||||
<id>ng_backup</id>
|
||||
<name>NG Backup</name>
|
||||
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
||||
<description><![CDATA[Release candidate for 1.0.0: scheduled, encrypted, incremental backups of the database, config and data
|
||||
directory to S3, SFTP, WebDAV, FTP, SMB or a local path, with restore of files and folders, of
|
||||
single users (via user_migration) and of the whole server onto a fresh installation. Tested
|
||||
thoroughly and usable alongside your current backup, but not yet a drop-in replacement for it.
|
||||
See the README for known limitations and the recovery kit before relying on it.]]></description>
|
||||
<version>0.9.7</version>
|
||||
<summary>Encrypted, incremental backups of your Nextcloud — files, database and config — with restore from the web interface</summary>
|
||||
<description>< a star on GitHub!**
|
||||
|
||||
Developed and sponsored by [Ensembia](https://www.ensembia.com). We can offer (paid) assistance if
|
||||
you need help with restoring a backup after a crash.]]></description>
|
||||
<version>1.1.0</version>
|
||||
<licence>agpl</licence>
|
||||
<author mail="dillard.blom@ensembia.com">Dillard Blom</author>
|
||||
<namespace>NgBackup</namespace>
|
||||
@@ -20,9 +56,10 @@ See the README for known limitations and the recovery kit before relying on it.]
|
||||
<website>https://github.com/dillardblom/ng_backup</website>
|
||||
<bugs>https://github.com/dillardblom/ng_backup/issues</bugs>
|
||||
<repository type="git">https://github.com/dillardblom/ng_backup</repository>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/1-backups.png</screenshot>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/2-restore.png</screenshot>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/3-settings.png</screenshot>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/v1.1.0/screenshots/1-backups.png</screenshot>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/v1.1.0/screenshots/2-restore.png</screenshot>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/v1.1.0/screenshots/3-settings.png</screenshot>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/v1.1.0/screenshots/4-bin.png</screenshot>
|
||||
<dependencies>
|
||||
<php min-version="8.1"/>
|
||||
<lib>sodium</lib>
|
||||
@@ -46,11 +83,15 @@ See the README for known limitations and the recovery kit before relying on it.]
|
||||
<command>OCA\NgBackup\Command\ListSnapshots</command>
|
||||
<command>OCA\NgBackup\Command\Browse</command>
|
||||
<command>OCA\NgBackup\Command\Restore</command>
|
||||
<command>OCA\NgBackup\Command\PauseJobs</command>
|
||||
<command>OCA\NgBackup\Command\Pin</command>
|
||||
<command>OCA\NgBackup\Command\Note</command>
|
||||
<command>OCA\NgBackup\Command\Prune</command>
|
||||
<command>OCA\NgBackup\Command\Retention</command>
|
||||
<command>OCA\NgBackup\Command\TargetAppendOnly</command>
|
||||
<command>OCA\NgBackup\Command\TargetTrustCurrent</command>
|
||||
<command>OCA\NgBackup\Command\TargetLimit</command>
|
||||
<command>OCA\NgBackup\Command\TargetSchedule</command>
|
||||
<command>OCA\NgBackup\Command\Trash</command>
|
||||
<command>OCA\NgBackup\Command\TrashRestore</command>
|
||||
<command>OCA\NgBackup\Command\KeySlots</command>
|
||||
@@ -59,6 +100,7 @@ See the README for known limitations and the recovery kit before relying on it.]
|
||||
<command>OCA\NgBackup\Command\KeySlotRemove</command>
|
||||
<command>OCA\NgBackup\Command\BackupUser</command>
|
||||
<command>OCA\NgBackup\Command\RestoreUser</command>
|
||||
<command>OCA\NgBackup\Command\ListUserExports</command>
|
||||
<command>OCA\NgBackup\Command\KeyImport</command>
|
||||
<command>OCA\NgBackup\Command\RestoreFull</command>
|
||||
<command>OCA\NgBackup\Command\Verify</command>
|
||||
|
||||
@@ -24,8 +24,12 @@ return [
|
||||
['name' => 'AdminApi#testTarget', 'url' => '/api/targets/{id}/test', 'verb' => 'POST'],
|
||||
['name' => 'AdminApi#setTargetLimit', 'url' => '/api/targets/{id}/limit', 'verb' => 'PUT'],
|
||||
['name' => 'AdminApi#startBackup', 'url' => '/api/backups', 'verb' => 'POST'],
|
||||
['name' => 'AdminApi#setSchedule', 'url' => '/api/schedule', 'verb' => 'PUT'],
|
||||
['name' => 'AdminApi#tick', 'url' => '/api/tick', 'verb' => 'POST'],
|
||||
['name' => 'AdminApi#setPauseHeavyJobs', 'url' => '/api/settings/pause-heavy-jobs', 'verb' => 'PUT'],
|
||||
['name' => 'AdminApi#setTargetSchedule', 'url' => '/api/targets/{id}/schedule', 'verb' => 'PUT'],
|
||||
['name' => 'AdminApi#snapshots', 'url' => '/api/targets/{targetId}/snapshots', 'verb' => 'GET'],
|
||||
['name' => 'AdminApi#setSnapshotNote', 'url' => '/api/targets/{targetId}/snapshots/{snapshotId}/note', 'verb' => 'PUT'],
|
||||
['name' => 'AdminApi#setSnapshotPinned', 'url' => '/api/targets/{targetId}/snapshots/{snapshotId}/pin', 'verb' => 'PUT'],
|
||||
['name' => 'AdminApi#browse', 'url' => '/api/targets/{targetId}/snapshots/{snapshotId}/browse', 'verb' => 'GET'],
|
||||
['name' => 'AdminApi#startRestore', 'url' => '/api/restores', 'verb' => 'POST'],
|
||||
],
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
# Disaster recovery: your server is gone
|
||||
|
||||
Your Nextcloud server is lost. You have a new, empty server. These are the steps to get your
|
||||
Nextcloud back from an NG Backup location. Do them in this order.
|
||||
|
||||
## What you need
|
||||
|
||||
- [ ] **One passphrase** of the old server, and if you have it, the **recovery kit** file
|
||||
(`ng_backup-kit.json`).
|
||||
- [ ] The **connection details of the backup location**: host, user, password or keys, bucket, folder.
|
||||
- [ ] A new server with the **same Nextcloud major version** as the old one (for example 34),
|
||||
a database (PostgreSQL, MySQL/MariaDB or SQLite; it may differ from the old one) and room for
|
||||
the whole backup plus a margin.
|
||||
- [ ] **Shell access** to the new server, to run `occ` (on a hosted Nextcloud without it, your
|
||||
provider runs these steps).
|
||||
- [ ] A list of the apps that are **not in the app store** (your own builds, apps copied in by
|
||||
hand). Keep their packages somewhere safe beforehand.
|
||||
|
||||
No kit? Since version 1.1.0 the passphrase is enough: the key is also stored on the location (in
|
||||
its `config` file), locked with the same passphrases. Skip step 4. The kit is still the safer
|
||||
copy: it helps when that file on the location is damaged. Keep it off the server, next to the
|
||||
passphrase.
|
||||
|
||||
## Steps
|
||||
|
||||
**1. Install a fresh Nextcloud** of the same major version. Create an admin account. Do not add
|
||||
users or files.
|
||||
|
||||
**2. Do not let background jobs run yet.** A new Nextcloud runs them on page visits (AJAX).
|
||||
Switch that off:
|
||||
|
||||
occ background:cron
|
||||
|
||||
As long as no cron is set up, nothing runs. Check that none is:
|
||||
|
||||
- Regular server: `sudo crontab -u www-data -l`. If it shows a line with `cron.php`, remove it
|
||||
for now with `sudo crontab -u www-data -e` (put it back in step 9).
|
||||
- Docker Compose with a cron service: `docker compose stop cron`.
|
||||
- Nextcloud All-in-One: cron runs inside the Nextcloud container; leave it. It does no harm here:
|
||||
the restore keeps NG Backup's own settings of the new server, so it does not start backups.
|
||||
|
||||
This keeps the new server quiet (no mail syncs, no notifications, no backups) until it is ready.
|
||||
|
||||
**3. Install NG Backup** (the same version or newer) and enable it.
|
||||
|
||||
**4. Import the backup key** (with a kit; without one, go to step 5). Copy the kit to the new
|
||||
server so the web server user can read it.
|
||||
|
||||
From your own computer:
|
||||
|
||||
scp ng_backup-kit.json root@newserver:/tmp/
|
||||
|
||||
Then on the server, one of:
|
||||
|
||||
chown www-data /tmp/ng_backup-kit.json # regular server
|
||||
docker cp /tmp/ng_backup-kit.json <container>:/tmp/ && \
|
||||
docker exec <container> chown www-data /tmp/ng_backup-kit.json # Docker
|
||||
|
||||
`<container>` is the name of your Nextcloud container: `docker ps` shows it (for example
|
||||
`nextcloud-aio-nextcloud`, or `myproject-nextcloud-1` with Docker Compose).
|
||||
|
||||
Import it; you type the passphrase yourself:
|
||||
|
||||
occ backup:key:import /tmp/ng_backup-kit.json
|
||||
|
||||
It shows the key fingerprint: it must match the one of your old server (on the recovery kit).
|
||||
|
||||
**5. Add the backup location** with the same details as before. On a large backup this first reads
|
||||
the index and shows how far it is, for example
|
||||
`Reading the index: 6,913/26,791 files (25%), about 12 min left`.
|
||||
|
||||
occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=... -o root=... --path=...
|
||||
|
||||
Without a kit (step 4 skipped), it finds the backups and asks for a passphrase of the old server;
|
||||
you type it yourself. It then shows the key fingerprint and the deletion delay of the old server.
|
||||
A wrong passphrase changes nothing: run the command again. Backups made before version 1.1.0 did
|
||||
not store the deletion delay on the location; it is then set to 7 days.
|
||||
|
||||
If it says that the location's **config file is gone**, the key is only in the recovery kit:
|
||||
import it (step 4), then add the location with `--repair-config` at the end of the same command.
|
||||
That writes the config again from the kit; it works for locations backed up by version 1.1.0 or
|
||||
newer.
|
||||
|
||||
If it says "No backup key yet, and no backups at this location", the connection details or
|
||||
`--path` differ from the old server's. Check them; do **not** run `backup:key:init`: that makes a
|
||||
brand new key, and your backups were encrypted with the old one. NG Backup then refuses the
|
||||
location ("created with a different key").
|
||||
|
||||
**Made a new key by mistake** (with `backup:key:init` or on the page) before adding the location?
|
||||
As long as that key was never used (no location added with it, no backup), it can make way for the
|
||||
key of your backups. NG Backup says so when it refuses the location. Add `--replace-unused-key` to
|
||||
`occ backup:key:import` (with the kit) or to `occ backup:target:add` (without the kit; it asks for
|
||||
a passphrase of the old server). The passphrases and kit of the unused key then stop working. A key
|
||||
that was used for a location is never replaced: its own backups depend on it. A key made with a
|
||||
version before 1.1.0 always counts as used (those versions did not keep track), also when it never
|
||||
protected a location; recover onto a fresh Nextcloud installation instead.
|
||||
|
||||
**6. List the restore points** and pick one, normally the newest:
|
||||
|
||||
occ backup:list offsite
|
||||
|
||||
**7. Start the restore, and let it install the apps.**
|
||||
|
||||
occ backup:restore:full offsite <snapshot> --install-apps
|
||||
|
||||
`--install-apps` first installs every app the old server had enabled from the app store (or
|
||||
switches it on, if it is part of Nextcloud). Tables of apps that are missing or were removed long
|
||||
ago are created from the backup, so they do not stop the restore.
|
||||
|
||||
At the end it lists the apps it could not install, usually your own builds that are not in the
|
||||
app store. Copy those into `custom_apps` and enable them (`occ app:enable <app>`); their data is
|
||||
already restored.
|
||||
|
||||
Backups made before 1.0.0-rc.2 do not record the structure of their tables. If the restore lists
|
||||
tables that do not exist here, install the apps they belong to and run it again, or leave them
|
||||
out with `--skip-missing-tables`.
|
||||
|
||||
**8. Let the restore run.** It restores the database, then the data directory, and takes hours for
|
||||
a large backup. Nextcloud is in maintenance mode meanwhile, so nobody and no background job
|
||||
writes into it; the restore switches it off when it completes. (If you had switched maintenance
|
||||
mode on yourself before, it stays on: switch it off with `occ maintenance:mode --off` when ready.)
|
||||
|
||||
If it stops halfway, maintenance mode stays on (the instance may be partly restored). Fix the cause,
|
||||
then switch maintenance mode off and run the restore again with `--force`, in one line (the message
|
||||
prints it): `occ maintenance:mode --off && occ backup:restore:full <location> <snapshot> --force`.
|
||||
It goes on where it stopped: data files that are already in place with the size and modification
|
||||
time of the backup are left, so only the rest comes from the location again (since 1.0.2; before,
|
||||
it started over). The database is restored again in full, which takes minutes. Leave out `--install-apps` this time, so that the instance is open only for the seconds until the
|
||||
restore has switched maintenance mode on again; apps the first run could not install are listed at
|
||||
the end of the restore, install those afterwards.
|
||||
If occ then says that Nextcloud or one of the apps requires an upgrade, the database is already
|
||||
the old server's: run `occ upgrade` first, then the same command with `--force`.
|
||||
|
||||
**If the backup itself is damaged** (the restore stops with "Decryption failed", "Blob
|
||||
verification failed" or a file that is missing on the location), try an older restore point
|
||||
first. If none is better, `--salvage` restores whatever can be read:
|
||||
|
||||
occ backup:restore:full offsite <snapshot> --install-apps --salvage
|
||||
|
||||
If a restore already stopped halfway, run it as in step 8: maintenance mode off, with `--force` and
|
||||
without `--install-apps`:
|
||||
|
||||
occ maintenance:mode --off && occ backup:restore:full offsite <snapshot> --salvage --force
|
||||
|
||||
- Files that cannot be read are left out and listed at the end; everything else is restored.
|
||||
- A full disk is not damage: the restore stops, also with `--salvage`, and says so. Make room and
|
||||
run the restore again as in step 8.
|
||||
- **On version 1.0.2, check the disk first if the reason is "Size mismatch".** There a full disk
|
||||
looks like damaged data: the message points at `--salvage`, and with `--salvage` the restore goes
|
||||
on and lists the remaining files. The backup itself is fine. Update NG Backup, make room and run
|
||||
the restore again as in step 8.
|
||||
- If the database cannot be restored (its dump is damaged, or it does not fit this
|
||||
installation), the database stays as the fresh installation made it, and the data directory
|
||||
and config of the backup go to a folder of their own in the data directory,
|
||||
`ngb-salvage-<snapshot>`. Each user's files are in `data/<user>/files` there: create the users
|
||||
again and copy their files back by hand (then `occ files:scan --all`). Calendars, contacts and
|
||||
shares are in the database and cannot be recovered this way.
|
||||
|
||||
The command then ends with exit code 2 instead of 0. The same option exists for
|
||||
`occ backup:restore`, to restore single folders.
|
||||
|
||||
**9. After the restore.** These worked in our tests:
|
||||
|
||||
occ upgrade
|
||||
occ maintenance:repair
|
||||
occ files:scan --all
|
||||
|
||||
- Check `config/config.php`: `trusted_domains`, `overwrite.cli.url` and `overwritehost` come from
|
||||
the old server; change them if the new server has another name. The database settings and the
|
||||
data directory are kept from the new installation.
|
||||
- From Nextcloud All-in-One to a regular server: also check the cache settings (`memcache.*`,
|
||||
`redis`), which point at AIO's containers.
|
||||
- Log in, open a few files, a calendar and a contact.
|
||||
- `occ files:scan` can report many folders as "Updated". That is expected: a restore does not yet put
|
||||
back the modification time of folders with files in them, so they show the time of the restore.
|
||||
The files themselves keep their time and content.
|
||||
- Empty folders only come back from backups made with version 1.0.3 or newer. With an older
|
||||
restore point they are gone: `occ files:scan` then removes them from the file cache, so it reports
|
||||
them as removed.
|
||||
- Start background jobs again (cron), as in step 2 in reverse.
|
||||
|
||||
**10. Point your domain at the new server** and check that clients reconnect.
|
||||
|
||||
**11. Make a backup of the new server** to the same location (`occ backup:run offsite`). The old
|
||||
server must stay off: two servers must never write to the same location.
|
||||
|
||||
Restored without a kit? Then backups wait until you have a new one: download it on the backup
|
||||
page (or `occ backup:key:kit --output=...`), store it off the server and confirm it
|
||||
(`occ backup:key:confirm`).
|
||||
|
||||
## Never
|
||||
|
||||
- Never let the old server write to the location again if it comes back. Switch its backups off
|
||||
first.
|
||||
- Never give your passphrase to anyone. Nobody helping you needs it.
|
||||
|
||||
## Help
|
||||
|
||||
Stuck? [Ensembia](https://www.ensembia.com), the developer of NG Backup, offers paid help with
|
||||
restoring a backup after a crash.
|
||||
@@ -79,7 +79,9 @@ occ backup:list
|
||||
Later runs upload only changed data. An unchanged run uploads (almost) nothing; the output shows
|
||||
the number of uploaded bytes, so check that value when you test incremental behaviour.
|
||||
|
||||
Schedule `occ backup:run` from system cron (or use the Administration settings, if you prefer).
|
||||
Set when each location is backed up in the Administration settings, or with
|
||||
`occ backup:target:schedule offsite 02:00` (once a day), `02:00/6` (every 6 hours from 02:00) or `off`.
|
||||
Locations are backed up one after the other.
|
||||
|
||||
## 4. Check the backup regularly
|
||||
|
||||
@@ -114,10 +116,16 @@ Needs the `user_migration` app. NG Backup asks before installing it:
|
||||
|
||||
```
|
||||
occ backup:user:backup offsite <uid>
|
||||
occ backup:user:list offsite [<uid>] # the exports and their manifest
|
||||
occ backup:user:restore offsite users/<uid>/<exportId> # replace (default)
|
||||
occ backup:user:restore offsite users/<uid>/<exportId> --mode=as-backup
|
||||
```
|
||||
|
||||
- `backup:user:restore` needs the manifest path that `backup:user:backup` prints at the end
|
||||
(`users/<uid>/<exportId>`); `backup:user:list` shows it again. A user id alone also works when
|
||||
that user has exactly one export on the location. With more (a `replace` restore adds a safety
|
||||
export), the command lists them and you pass one.
|
||||
|
||||
- If `user_migration` is missing, the command asks to install and enable it. For non-interactive
|
||||
use, add `--install-user-migration` to approve that explicitly. The command stops after the
|
||||
installation; run it once more.
|
||||
@@ -136,6 +144,8 @@ occ backup:user:restore offsite users/<uid>/<exportId> --mode=as-backup
|
||||
|
||||
## 7. Disaster recovery: restore the whole server
|
||||
|
||||
Server lost? Follow the step-by-step [DISASTER-RECOVERY.md](DISASTER-RECOVERY.md).
|
||||
|
||||
Use a fresh Nextcloud installation of the same major version, with NG Backup installed and
|
||||
the same location available. For a test, use a **copy** of the location: two servers writing to one
|
||||
location break each other's catalog.
|
||||
@@ -147,12 +157,20 @@ occ backup:list offsite # the snapshots must be lis
|
||||
occ backup:restore:full offsite <snapshot>
|
||||
```
|
||||
|
||||
Without the kit, leave out `backup:key:import`: `backup:target:add` finds the backups and takes
|
||||
the key from the location with a passphrase (`--passphrase-file=/path/passphrase`, or it asks).
|
||||
Backups then wait until a new kit is downloaded and confirmed.
|
||||
|
||||
The command refuses on an installation that already has more than one user, and asks before it
|
||||
starts; `--force` skips both (also needed to rerun it after it stopped halfway).
|
||||
|
||||
Afterwards, check that users, files and the database are complete, and run a backup of the restored
|
||||
server. The restored installation keeps its own app configuration and secret.
|
||||
|
||||
Stuck? [Ensembia](https://www.ensembia.com), the developer of NG Backup, offers paid help with
|
||||
restoring a backup after a crash. You keep your passphrase to yourself: it is never needed by
|
||||
anyone helping you.
|
||||
|
||||
## 8. Retention
|
||||
|
||||
```
|
||||
@@ -161,8 +179,13 @@ occ backup:prune offsite --dry-run # show first
|
||||
occ backup:prune offsite # apply the policy; forgotten snapshots go to the trash first
|
||||
occ backup:trash # list the trash
|
||||
occ backup:trash:restore offsite <id>
|
||||
occ backup:pin offsite <id> # keep this restore point; --unpin to undo
|
||||
occ backup:note offsite <id> "text" # a note you can change later; without text it is removed
|
||||
```
|
||||
|
||||
The retention policy leaves a pinned restore point alone, and a pinned one does not take the place of
|
||||
a daily, weekly or monthly one. Pin and note are stored on the location, so a new server sees them too.
|
||||
|
||||
Snapshots are kept according to the policy, and their data is deleted only after the deletion delay
|
||||
(set with `backup:key:init --delete-delay-days`). `occ backup:prune offsite --dry-run` shows what would be
|
||||
removed without changing anything.
|
||||
|
||||
@@ -72,6 +72,10 @@ final class ExternalStorageFactory {
|
||||
throw new BackendException("Auth mechanism $authId does not fit backend $backendId");
|
||||
}
|
||||
|
||||
if (is_string($options['private_key'] ?? null)) {
|
||||
$options['private_key'] = self::normalizePrivateKey($options['private_key']);
|
||||
}
|
||||
|
||||
$config = new StorageConfig();
|
||||
$config->setBackend($backend);
|
||||
$config->setAuthMechanism($auth);
|
||||
@@ -86,4 +90,22 @@ final class ExternalStorageFactory {
|
||||
$storage = $backend->wrapStorage($storage);
|
||||
return $auth->wrapStorage($storage);
|
||||
}
|
||||
|
||||
/**
|
||||
* An OpenSSH-format private key with its line breaks restored. Up to Nextcloud 34
|
||||
* (phpseclib 2) a key whose newlines were lost or turned into spaces, e.g. on its way through
|
||||
* a shell or a form, still loaded; Nextcloud 35 (phpseclib 3) refuses it with "Unable to read
|
||||
* key" (#24). The key itself is not changed, so it is fixed when it is used, not when stored.
|
||||
* Other formats (PEM, PKCS#8) still load as they are and are returned unchanged.
|
||||
*/
|
||||
public static function normalizePrivateKey(string $key): string {
|
||||
if (!preg_match('/-----BEGIN OPENSSH PRIVATE KEY-----(.*?)-----END OPENSSH PRIVATE KEY-----/s', $key, $m)) {
|
||||
return $key;
|
||||
}
|
||||
$body = (string)preg_replace('/\s+/', '', $m[1]);
|
||||
if (!preg_match('#^[A-Za-z0-9+/]+=*$#', $body)) {
|
||||
return $key;
|
||||
}
|
||||
return "-----BEGIN OPENSSH PRIVATE KEY-----\n" . chunk_split($body, 70, "\n") . "-----END OPENSSH PRIVATE KEY-----\n";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
/**
|
||||
* A backend that can read many byte ranges ahead of use, several at a time over connections of
|
||||
* its own (a full restore, where one connection at a time is bound by the round trip).
|
||||
*/
|
||||
interface IReadsAhead {
|
||||
/** Whether readAhead() is worth planning for: false when it would return null anyway. */
|
||||
public function canReadAhead(): bool;
|
||||
|
||||
/**
|
||||
* The ranges in the order given, each as a local stream at offset 0 holding its bytes, read a
|
||||
* few ranges ahead of the one the caller is at. A stream is closed when the generator is
|
||||
* resumed. Null for a range that could not be read this way (a missing object, a server or
|
||||
* connection error): the caller reads it the usual way, which reports why.
|
||||
*
|
||||
* @param list<array{0:string, 1:int, 2:int}> $ranges [path, offset, length] each
|
||||
* @param string $spool an existing local directory for the files that hold the ranges
|
||||
* @return \Generator<int, resource|null>|null by position in $ranges; null when this backend
|
||||
* cannot read ahead now
|
||||
*/
|
||||
public function readAhead(array $ranges, string $spool): ?\Generator;
|
||||
}
|
||||
@@ -67,6 +67,9 @@ final class LocalBackend implements IBackend {
|
||||
if ($fh === false) {
|
||||
throw new BackendException('Not found: ' . $path);
|
||||
}
|
||||
// Packs are read in large pieces: one request each, not 8 KiB at a time (see Repository::openForReading()).
|
||||
stream_set_read_buffer($fh, 0);
|
||||
stream_set_chunk_size($fh, 4 * 1048576);
|
||||
return $fh;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
/**
|
||||
* Reads byte ranges of files on an SFTP location ahead of use, over several connections at once,
|
||||
* into local files (StorageBackend::readAhead(), for a full restore).
|
||||
*
|
||||
* A restore that reads its packs as it goes is bound by the round trip: from a Hetzner Storage
|
||||
* Box to Canada (RTT 122 ms) it reached about 2.8 MiB/s, with the PHP process mostly waiting.
|
||||
* Here every connection reads its own range with DEPTH requests in flight, ahead of the restore. PHP runs one thing at a time: while it waits for an answer on one connection, the
|
||||
* answers of the others arrive in the kernel's buffers, so the connections still transfer at the
|
||||
* same time.
|
||||
*
|
||||
* Uses the same internal phpseclib 2 methods as SftpReadStream (see SftpReadStream::connection()).
|
||||
* The connections are this class's own, never the storage's: no other request goes over them.
|
||||
* A range that fails (missing file, server error, a connection that breaks) comes out as null,
|
||||
* for the caller to read the usual way; a broken connection is not used again.
|
||||
*/
|
||||
final class SftpRangeFetcher {
|
||||
/**
|
||||
* Connections a full restore uses, unless the app setting restore_sftp_connections says
|
||||
* otherwise. One: a Hetzner Storage Box gave about 14 MiB/s on one connection with DEPTH
|
||||
* requests in flight, and 4-6 MiB/s on 2 or 4 (it reads from disks, and parallel readers make
|
||||
* them seek). On a line with packet loss to a server that handles parallel reads well, 4
|
||||
* connections did better (21 against 14 MiB/s).
|
||||
*/
|
||||
public const DEFAULT_CONNECTIONS = 1;
|
||||
public const MAX_CONNECTIONS = 8;
|
||||
/** READ requests in flight per connection. */
|
||||
public const DEPTH = 32;
|
||||
/** Ranges read ahead of the one the caller is at, in flight or waiting in the spool. */
|
||||
public const AHEAD = 8;
|
||||
/**
|
||||
* Seconds ready() waits for any connection to answer before it waits for each in turn: short,
|
||||
* in case a connection holds an answer where ready() does not look.
|
||||
*/
|
||||
private const SELECT_TIMEOUT = 1;
|
||||
/** See SftpReadStream::CHUNK and MIN_CHUNK. */
|
||||
private const CHUNK = 261120;
|
||||
private const MIN_CHUNK = 32768;
|
||||
|
||||
private const FXP_OPEN = 3;
|
||||
private const FXP_CLOSE = 4;
|
||||
private const FXP_READ = 5;
|
||||
private const FXP_STATUS = 101;
|
||||
private const FXP_HANDLE = 102;
|
||||
private const FXP_DATA = 103;
|
||||
private const FXF_READ = 1;
|
||||
private const FX_OK = 0;
|
||||
|
||||
/** Connections that broke and are replaced, at most, in one restore. */
|
||||
public const MAX_RECONNECTS = 20;
|
||||
|
||||
private const OPEN = 0;
|
||||
private const READ = 1;
|
||||
private const CLOSE = 2;
|
||||
|
||||
/** @var array<int, int> READ size per connection, lowered when a server answers less than asked */
|
||||
private array $chunks = [];
|
||||
/** @var array<int, SftpRangeJob> the range each busy connection reads */
|
||||
private array $jobs = [];
|
||||
/** @var array<int, resource|null> ranges read (or failed) and not yet given to the caller */
|
||||
private array $done = [];
|
||||
private int $requestId = 0x4e520000;
|
||||
/** Connections replaced so far (see MAX_RECONNECTS). */
|
||||
public int $reconnects = 0;
|
||||
|
||||
/**
|
||||
* @param array<int, object> $connections phpseclib 2 connections with their SFTP channel open
|
||||
* @param list<array{0:string, 1:int, 2:int}> $ranges paths below $root
|
||||
*/
|
||||
private function __construct(
|
||||
private array $connections,
|
||||
private string $root,
|
||||
private array $ranges,
|
||||
private string $spool,
|
||||
private ?\Closure $connect = null,
|
||||
) {
|
||||
foreach (array_keys($connections) as $c) {
|
||||
$this->chunks[$c] = self::CHUNK;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* See IReadsAhead::readAhead(). The connections are ended when the generator ends.
|
||||
*
|
||||
* @param list<object> $connections
|
||||
* @param list<array{0:string, 1:int, 2:int}> $ranges
|
||||
* @param (\Closure():?object)|null $connect a new connection in place of one that broke
|
||||
* (null when it cannot be made)
|
||||
* @return \Generator<int, resource|null>
|
||||
*/
|
||||
public static function fetch(array $connections, string $root, array $ranges, string $spool, ?\Closure $connect = null): \Generator {
|
||||
$fetcher = new self($connections, rtrim($root, '/'), $ranges, $spool, $connect);
|
||||
try {
|
||||
yield from $fetcher->run();
|
||||
} finally {
|
||||
$fetcher->end();
|
||||
}
|
||||
}
|
||||
|
||||
/** @return \Generator<int, resource|null> */
|
||||
private function run(): \Generator {
|
||||
$count = count($this->ranges);
|
||||
$next = 0;
|
||||
for ($want = 0; $want < $count; $want++) {
|
||||
while (!array_key_exists($want, $this->done)) {
|
||||
foreach (array_keys($this->connections) as $c) {
|
||||
if (!isset($this->jobs[$c]) && $next < $count && $next < $want + self::AHEAD) {
|
||||
$this->start($c, $next++);
|
||||
}
|
||||
}
|
||||
if ($this->jobs === [] && !array_key_exists($want, $this->done)) {
|
||||
// no connection left: the rest the usual way
|
||||
$this->done[$want] = null;
|
||||
$next = max($next, $want + 1);
|
||||
break;
|
||||
}
|
||||
foreach ($this->ready() as $c) {
|
||||
$this->step($c);
|
||||
}
|
||||
}
|
||||
$fh = $this->done[$want];
|
||||
unset($this->done[$want]);
|
||||
try {
|
||||
yield $want => $fh;
|
||||
} finally {
|
||||
if (is_resource($fh)) {
|
||||
fclose($fh);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The busy connections that have an answer waiting, so that a connection that waits for a
|
||||
* lost packet to be sent again does not hold up the others; all of them when the connections
|
||||
* do not tell (then they are taken in turn).
|
||||
*
|
||||
* @return list<int>
|
||||
*/
|
||||
private function ready(): array {
|
||||
$sockets = [];
|
||||
$ready = [];
|
||||
foreach ($this->jobs as $c => $job) {
|
||||
$sftp = $this->connections[$c];
|
||||
$vars = get_object_vars($sftp); // only what is public: phpseclib 2's are
|
||||
// Answers phpseclib has read already: the socket does not show them. Its request
|
||||
// buffer holds whole answers read while it waited for another; its packet buffer and
|
||||
// channel buffers hold what came in with the last packet it read (small answers, such
|
||||
// as a CLOSE and the next OPEN, often arrive together).
|
||||
$buffered = $vars['requestBuffer'] ?? null;
|
||||
$channels = $vars['channel_buffers'] ?? null;
|
||||
if ((is_array($buffered) && isset($job->queue[0]) && array_key_exists($job->queue[0][0], $buffered))
|
||||
|| (is_string($vars['packet_buffer'] ?? null) && self::wholePacket($vars['packet_buffer']))
|
||||
|| (is_array($channels) && array_filter($channels) !== [])) {
|
||||
$ready[] = $c;
|
||||
continue;
|
||||
}
|
||||
$socket = $vars['fsock'] ?? null;
|
||||
if (!is_resource($socket)) {
|
||||
return array_keys($this->jobs);
|
||||
}
|
||||
$sockets[$c] = $socket;
|
||||
}
|
||||
if ($ready !== [] || $sockets === []) {
|
||||
return $ready;
|
||||
}
|
||||
$read = $sockets;
|
||||
$write = null;
|
||||
$except = null;
|
||||
// Data in PHP's own buffer of a socket counts as readable. Nothing within the time:
|
||||
// each in turn, where phpseclib's own timeout applies.
|
||||
$n = @stream_select($read, $write, $except, self::SELECT_TIMEOUT);
|
||||
return $n === false || $n === 0 ? array_keys($this->jobs) : array_keys($read);
|
||||
}
|
||||
|
||||
/** Whether $buffer starts with a whole SFTP packet (length, then that many bytes). */
|
||||
private static function wholePacket(string $buffer): bool {
|
||||
$length = strlen($buffer) >= 4 ? unpack('N', $buffer) : false;
|
||||
return is_array($length) && strlen($buffer) >= 4 + $length[1];
|
||||
}
|
||||
|
||||
private function start(int $c, int $i): void {
|
||||
[$path, $offset, $length] = $this->ranges[$i];
|
||||
$fh = $this->spoolFile();
|
||||
if ($fh === null) {
|
||||
$this->done[$i] = null;
|
||||
return;
|
||||
}
|
||||
if ($length <= 0) {
|
||||
$this->done[$i] = $fh;
|
||||
return;
|
||||
}
|
||||
$this->jobs[$c] = new SftpRangeJob($i, $fh, $offset, $offset + $length);
|
||||
try {
|
||||
$remote = $this->connections[$c]->_realpath($this->root . '/' . ltrim($path, '/'));
|
||||
if (!is_string($remote)) {
|
||||
$this->finish($c);
|
||||
return;
|
||||
}
|
||||
$this->send($c, self::FXP_OPEN, pack('Na*N2', strlen($remote), $remote, self::FXF_READ, 0), self::OPEN);
|
||||
} catch (\Throwable) {
|
||||
$this->drop($c);
|
||||
}
|
||||
}
|
||||
|
||||
/** Take the oldest answer on connection $c, then keep its requests in flight. */
|
||||
private function step(int $c): void {
|
||||
$job = $this->jobs[$c];
|
||||
[$id, $kind, $offset, $length] = array_shift($job->queue) ?? throw new \LogicException('no request in flight');
|
||||
$sftp = $this->connections[$c];
|
||||
try {
|
||||
$response = $sftp->_get_sftp_packet($id);
|
||||
if (!is_string($response)) {
|
||||
throw new BackendException('SFTP connection lost');
|
||||
}
|
||||
$type = $sftp->packet_type;
|
||||
if ($kind === self::OPEN) {
|
||||
if ($type === self::FXP_HANDLE) {
|
||||
$job->handle = substr($response, 4);
|
||||
} else {
|
||||
$job->failed = true; // missing, or no permission
|
||||
}
|
||||
} elseif ($kind === self::READ) {
|
||||
if (!$job->failed) {
|
||||
$this->received($c, $type, $response, $offset, $length);
|
||||
}
|
||||
} else {
|
||||
$status = $type === self::FXP_STATUS && strlen($response) >= 4 ? unpack('N', $response) : false;
|
||||
$closed = is_array($status) && $status[1] === self::FX_OK;
|
||||
$this->finish($c);
|
||||
if (!$closed) {
|
||||
// handles might pile up on the server: not used again
|
||||
$this->drop($c);
|
||||
}
|
||||
return;
|
||||
}
|
||||
$this->refill($c);
|
||||
} catch (\Throwable) {
|
||||
$this->drop($c);
|
||||
}
|
||||
}
|
||||
|
||||
private function received(int $c, int $type, string $response, int $offset, int $length): void {
|
||||
$job = $this->jobs[$c];
|
||||
$declared = strlen($response) >= 4 ? unpack('N', $response) : false;
|
||||
$n = is_array($declared) ? $declared[1] : 0;
|
||||
if ($type !== self::FXP_DATA || $n <= 0 || $n !== strlen($response) - 4 || $n > $length) {
|
||||
// end of file before the end of the range, a server error or a malformed answer
|
||||
$job->failed = true;
|
||||
return;
|
||||
}
|
||||
if (fseek($job->fh, $offset - $job->start) !== 0 || fwrite($job->fh, substr($response, 4)) !== $n) {
|
||||
$job->failed = true;
|
||||
return;
|
||||
}
|
||||
$job->got += $n;
|
||||
if ($n < $length) {
|
||||
// a server that caps a READ: ask for the rest, and no more than it gives from now on
|
||||
$job->redo[] = [$offset + $n, $length - $n];
|
||||
$this->chunks[$c] = max(self::MIN_CHUNK, min($this->chunks[$c], $n));
|
||||
}
|
||||
}
|
||||
|
||||
/** Send READs until DEPTH are in flight; when nothing is left to ask for, close the file. */
|
||||
private function refill(int $c): void {
|
||||
$job = $this->jobs[$c];
|
||||
if ($job->handle !== '' && !$job->failed) {
|
||||
while (count($job->queue) < self::DEPTH) {
|
||||
if ($job->redo !== []) {
|
||||
[$offset, $length] = array_shift($job->redo);
|
||||
} elseif ($job->next < $job->end) {
|
||||
$offset = $job->next;
|
||||
$length = min($this->chunks[$c], $job->end - $offset);
|
||||
$job->next += $length;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
$packet = pack('Na*N3', strlen($job->handle), $job->handle, intdiv($offset, 4294967296), $offset % 4294967296, $length);
|
||||
$this->send($c, self::FXP_READ, $packet, self::READ, $offset, $length);
|
||||
}
|
||||
}
|
||||
if ($job->queue !== []) {
|
||||
return;
|
||||
}
|
||||
if ($job->handle === '') {
|
||||
$this->finish($c); // not opened: nothing to close
|
||||
return;
|
||||
}
|
||||
$this->send($c, self::FXP_CLOSE, pack('Na*', strlen($job->handle), $job->handle), self::CLOSE);
|
||||
}
|
||||
|
||||
private function send(int $c, int $type, string $packet, int $kind, int $offset = 0, int $length = 0): void {
|
||||
$this->requestId = $this->requestId >= 0x7fffffff ? 0x4e520000 : $this->requestId + 1;
|
||||
if (!$this->connections[$c]->_send_sftp_packet($type, $packet, $this->requestId)) {
|
||||
throw new BackendException('SFTP connection failed while sending a request');
|
||||
}
|
||||
$this->jobs[$c]->queue[] = [$this->requestId, $kind, $offset, $length];
|
||||
}
|
||||
|
||||
/** The range of connection $c is done: its stream for the caller, or null when it failed. */
|
||||
private function finish(int $c): void {
|
||||
$job = $this->jobs[$c];
|
||||
unset($this->jobs[$c]);
|
||||
if ($job->failed || $job->got !== $job->end - $job->start || !rewind($job->fh)) {
|
||||
self::closeFile($job->fh);
|
||||
$this->done[$job->i] = null;
|
||||
return;
|
||||
}
|
||||
$this->done[$job->i] = $job->fh;
|
||||
}
|
||||
|
||||
/** A connection in an unknown state: its range fails, and it is not used again. */
|
||||
private function drop(int $c): void {
|
||||
if (isset($this->jobs[$c])) {
|
||||
$job = $this->jobs[$c];
|
||||
unset($this->jobs[$c]);
|
||||
self::closeFile($job->fh);
|
||||
$this->done[$job->i] = null;
|
||||
}
|
||||
self::disconnect($this->connections[$c]);
|
||||
unset($this->connections[$c]);
|
||||
// A long restore outlives connections (a Storage Box drops one now and then): without a
|
||||
// new one, the rest would be read the slow way.
|
||||
if ($this->connect === null || $this->reconnects >= self::MAX_RECONNECTS) {
|
||||
return;
|
||||
}
|
||||
$this->reconnects++;
|
||||
try {
|
||||
$new = ($this->connect)();
|
||||
} catch (\Throwable) {
|
||||
$new = null;
|
||||
}
|
||||
if (function_exists('OCP\Log\logger')) {
|
||||
\OCP\Log\logger('ng_backup')->info($new === null
|
||||
? 'Reading ahead: an SFTP connection broke and no new one could be made'
|
||||
: 'Reading ahead: an SFTP connection broke; reading on with a new one');
|
||||
}
|
||||
if ($new !== null) {
|
||||
$key = max(array_merge(array_keys($this->chunks), [-1])) + 1;
|
||||
$this->connections[$key] = $new;
|
||||
$this->chunks[$key] = self::CHUNK;
|
||||
}
|
||||
}
|
||||
|
||||
/** @return resource|null a new file in the spool, already unlinked: nothing is left behind if the process dies */
|
||||
private function spoolFile() {
|
||||
$path = $this->spool . '/' . bin2hex(random_bytes(8));
|
||||
$fh = @fopen($path, 'x+b');
|
||||
if ($fh === false) {
|
||||
return null;
|
||||
}
|
||||
@unlink($path);
|
||||
return $fh;
|
||||
}
|
||||
|
||||
private function end(): void {
|
||||
foreach ($this->jobs as $job) {
|
||||
self::closeFile($job->fh);
|
||||
}
|
||||
foreach ($this->done as $fh) {
|
||||
if (is_resource($fh)) {
|
||||
fclose($fh);
|
||||
}
|
||||
}
|
||||
$this->jobs = [];
|
||||
$this->done = [];
|
||||
foreach ($this->connections as $sftp) {
|
||||
self::disconnect($sftp);
|
||||
}
|
||||
$this->connections = [];
|
||||
}
|
||||
|
||||
private static function closeFile(mixed $fh): void {
|
||||
if (is_resource($fh)) {
|
||||
fclose($fh);
|
||||
}
|
||||
}
|
||||
|
||||
private static function disconnect(object $sftp): void {
|
||||
try {
|
||||
if (is_callable([$sftp, 'disconnect'])) {
|
||||
$sftp->disconnect();
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Backend;
|
||||
|
||||
/**
|
||||
* One range that SftpRangeFetcher reads on one connection.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
final class SftpRangeJob {
|
||||
public string $handle = '';
|
||||
/** Offset of the next READ to send. */
|
||||
public int $next;
|
||||
/** @var list<array{int, int}> offset and length of parts to ask for again (after a short answer) */
|
||||
public array $redo = [];
|
||||
/** @var list<array{int, int, int, int}> request id, kind, offset and length, in the order sent */
|
||||
public array $queue = [];
|
||||
/** Bytes received. */
|
||||
public int $got = 0;
|
||||
public bool $failed = false;
|
||||
|
||||
/** @var resource the spool file that holds the range */
|
||||
public $fh;
|
||||
|
||||
/** @param resource $fh */
|
||||
public function __construct(
|
||||
public int $i,
|
||||
$fh,
|
||||
public int $start,
|
||||
public int $end,
|
||||
) {
|
||||
$this->fh = $fh;
|
||||
$this->next = $start;
|
||||
}
|
||||
}
|
||||
@@ -144,9 +144,9 @@ final class SftpReadStream {
|
||||
|
||||
/**
|
||||
* The storage's phpseclib connection, with its SFTP channel open, when it offers what these
|
||||
* streams need; null otherwise.
|
||||
* streams (and SftpRangeFetcher) need; null otherwise.
|
||||
*/
|
||||
private static function connection(object $storage): ?object {
|
||||
public static function connection(object $storage): ?object {
|
||||
if (!is_callable([$storage, 'getConnection']) || !is_callable([$storage, 'getRoot'])) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ use OCP\Files\Storage\IWriteStreamStorage;
|
||||
* IBackend on top of a Nextcloud storage object (from files_external or core), below a base
|
||||
* directory on that storage.
|
||||
*/
|
||||
final class StorageBackend implements IBackend {
|
||||
final class StorageBackend implements IBackend, IReadsAhead {
|
||||
/** Object stores write whole objects atomically; other storages get a temp name + rename. */
|
||||
private bool $atomicPut;
|
||||
/** SFTP: read through SftpReadStream, which keeps several requests in flight. */
|
||||
@@ -23,12 +23,19 @@ final class StorageBackend implements IBackend {
|
||||
private bool $s3Reads;
|
||||
/** S3 requests in flight at once for getMany() and getRanges(). */
|
||||
public const S3_CONCURRENCY = 16;
|
||||
/**
|
||||
* Ranges readAhead() fetches at once over S3, each straight into a file in the spool (a range
|
||||
* is at most ReadAheadPlan::MAX_RANGE: the spool holds at most this many times that).
|
||||
*/
|
||||
public const S3_READ_AHEAD = 8;
|
||||
/** Most ranges, and bytes, getRanges() holds in memory at once. */
|
||||
private const RANGES_LOT = 64;
|
||||
private const RANGES_LOT_BYTES = 64 * 1048576;
|
||||
/** Seconds without a request after which an SFTP connection is checked before it is used. */
|
||||
public const IDLE_CHECK = 60;
|
||||
private float $lastUsed;
|
||||
/** Connections readAhead() opens (see SftpRangeFetcher::DEFAULT_CONNECTIONS). */
|
||||
public int $readAheadConnections = SftpRangeFetcher::DEFAULT_CONNECTIONS;
|
||||
/** Times the storage was built anew after its connection died (for tests and diagnostics). */
|
||||
public int $reconnects = 0;
|
||||
|
||||
@@ -117,7 +124,13 @@ final class StorageBackend implements IBackend {
|
||||
// null: not supported or failing for now; false: not opened, which need not mean
|
||||
// missing. Either way the storage's own stream decides.
|
||||
}
|
||||
$fh = $this->storage->fopen($this->abs($path), 'r');
|
||||
try {
|
||||
$fh = $this->storage->fopen($this->abs($path), 'r');
|
||||
} catch (\Exception $e) {
|
||||
// Some storages throw for what is not a missing file (SMB: a connection that broke).
|
||||
// As a BackendException, the caller reads once more and --salvage can go on.
|
||||
throw new BackendException('Cannot read ' . $path . ': ' . $e->getMessage(), 0, $e);
|
||||
}
|
||||
if ($fh === false) {
|
||||
throw new BackendException('Not found: ' . $path);
|
||||
}
|
||||
@@ -126,7 +139,7 @@ final class StorageBackend implements IBackend {
|
||||
|
||||
/**
|
||||
* Over SFTP and S3 several at a time (SftpReadStream::readMany(), s3Fetch()); elsewhere one
|
||||
* after the other.
|
||||
* after the other, over SMB in fewer round trips (smbWhole()).
|
||||
*/
|
||||
public function getMany(array $paths): \Generator {
|
||||
$size = $this->s3Reads ? 16 * self::S3_CONCURRENCY : 8 * SftpReadStream::BATCH;
|
||||
@@ -142,7 +155,8 @@ final class StorageBackend implements IBackend {
|
||||
$fast = $fast === null ? null : array_combine($chunk, array_map(fn (string $a): ?string => $fast[$a] ?? null, $abs));
|
||||
}
|
||||
foreach ($chunk as $path) {
|
||||
$data = $fast[$path] ?? null;
|
||||
// over SMB in fewer round trips, one after the other (smbWhole())
|
||||
$data = $fast[$path] ?? $this->smbWhole($this->abs($path));
|
||||
if ($data === null) {
|
||||
// not read in the batch (or no batches here): the usual way, which also
|
||||
// reports a missing file as such
|
||||
@@ -209,15 +223,11 @@ final class StorageBackend implements IBackend {
|
||||
*/
|
||||
private function s3Fetch(array $requests): ?array {
|
||||
try {
|
||||
/** @psalm-suppress UndefinedMethod, UndefinedThisPropertyFetch */
|
||||
[$client, $bucket, $sse] = (function (): array {
|
||||
$sse = method_exists($this, 'getServerSideEncryptionParameters') ? $this->getServerSideEncryptionParameters()
|
||||
: (method_exists($this, 'getSSECParameters') ? $this->getSSECParameters() : []);
|
||||
return [$this->getConnection(), $this->getBucket(), $sse];
|
||||
})->call($this->storage);
|
||||
if (!$client instanceof \Aws\S3\S3Client || !is_string($bucket)) {
|
||||
$s3 = $this->s3Client();
|
||||
if ($s3 === null) {
|
||||
return null;
|
||||
}
|
||||
[$client, $bucket, $sse] = $s3;
|
||||
$results = [];
|
||||
$keys = array_keys($requests); // the pool numbers results 0, 1, ...: mapped back below
|
||||
$commands = (function () use ($requests, $client, $bucket, $sse): \Generator {
|
||||
@@ -243,11 +253,174 @@ final class StorageBackend implements IBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* files_external's S3 client, bucket and server-side encryption parameters; null when the
|
||||
* storage is not as expected.
|
||||
*
|
||||
* @return array{0: \Aws\S3\S3Client, 1: string, 2: array}|null
|
||||
* @psalm-suppress UndefinedClass the AWS SDK comes with Nextcloud, not with this app
|
||||
*/
|
||||
private function s3Client(): ?array {
|
||||
/** @psalm-suppress UndefinedMethod, UndefinedThisPropertyFetch */
|
||||
[$client, $bucket, $sse] = (function (): array {
|
||||
$sse = method_exists($this, 'getServerSideEncryptionParameters') ? $this->getServerSideEncryptionParameters()
|
||||
: (method_exists($this, 'getSSECParameters') ? $this->getSSECParameters() : []);
|
||||
return [$this->getConnection(), $this->getBucket(), $sse];
|
||||
})->call($this->storage);
|
||||
return $client instanceof \Aws\S3\S3Client && is_string($bucket) ? [$client, $bucket, is_array($sse) ? $sse : []] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* readAhead() over S3: S3_READ_AHEAD ranges at a time, fetched together straight into files in
|
||||
* the spool, then handed out in order. One request per range instead of one per blob (with a
|
||||
* new HTTPS connection each), and several at once: a restore is no longer bound by the round
|
||||
* trip. A range that could not be fetched is handed out as null (the caller reads it the
|
||||
* usual way, which reports why).
|
||||
*
|
||||
* @param list<array{0:string, 1:int, 2:int}> $ranges
|
||||
* @return \Generator<int, resource|null>|null
|
||||
* @psalm-suppress UndefinedClass the AWS SDK comes with Nextcloud, not with this app
|
||||
*/
|
||||
private function s3ReadAhead(array $ranges, string $spool): ?\Generator {
|
||||
// before anything is fetched: an invalid path throws
|
||||
$keys = array_map(fn (array $r): string => trim($this->abs($r[0]), '/'), $ranges);
|
||||
try {
|
||||
$s3 = $this->s3Client();
|
||||
} catch (\Throwable) {
|
||||
$s3 = null;
|
||||
}
|
||||
if ($s3 === null) {
|
||||
return null;
|
||||
}
|
||||
[$client, $bucket, $sse] = $s3;
|
||||
$this->idleCheck();
|
||||
return (static function () use ($client, $bucket, $sse, $ranges, $keys, $spool): \Generator {
|
||||
$prefix = rtrim($spool, '/') . '/s3-' . bin2hex(random_bytes(4)) . '-';
|
||||
foreach (array_chunk(array_keys($ranges), self::S3_READ_AHEAD) as $lot) {
|
||||
$files = [];
|
||||
$commands = [];
|
||||
foreach ($lot as $n => $i) {
|
||||
$files[$i] = $prefix . $i;
|
||||
$commands[$n] = $client->getCommand('GetObject', ['Bucket' => $bucket, 'Key' => $keys[$i],
|
||||
'Range' => 'bytes=' . $ranges[$i][1] . '-' . ($ranges[$i][1] + $ranges[$i][2] - 1), 'SaveAs' => $files[$i]] + $sse);
|
||||
}
|
||||
$fetched = [];
|
||||
try {
|
||||
(new \Aws\CommandPool($client, $commands, [
|
||||
'concurrency' => self::S3_READ_AHEAD,
|
||||
'fulfilled' => function (mixed $result, int $n) use (&$fetched, $lot): void {
|
||||
$fetched[$lot[$n]] = true;
|
||||
},
|
||||
'rejected' => static function (): void {
|
||||
},
|
||||
]))->promise()->wait();
|
||||
} catch (\Throwable) {
|
||||
// what was not fetched is read the usual way
|
||||
}
|
||||
try {
|
||||
foreach ($lot as $i) {
|
||||
clearstatcache(true, $files[$i]);
|
||||
$fh = isset($fetched[$i]) && @filesize($files[$i]) === $ranges[$i][2] ? @fopen($files[$i], 'rb') : false;
|
||||
@unlink($files[$i]); // the open stream keeps its data
|
||||
yield $i => is_resource($fh) ? $fh : null;
|
||||
if (is_resource($fh)) {
|
||||
fclose($fh);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
// stopped halfway: the rest of this lot
|
||||
foreach ($files as $file) {
|
||||
@unlink($file);
|
||||
}
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Over SFTP: SftpRangeFetcher, on readAheadConnections new connections of its own (built with the
|
||||
* reconnect function), so the storage's connection stays free for other requests. Over S3:
|
||||
* S3_READ_AHEAD ranges at a time through the S3 client (s3ReadAhead()). Elsewhere not (yet):
|
||||
* null.
|
||||
*/
|
||||
public function canReadAhead(): bool {
|
||||
return ($this->sftpReads && $this->reconnect !== null) || $this->s3Reads || $this->smbNative();
|
||||
}
|
||||
|
||||
private function smbNative(): bool {
|
||||
return $this->storage instanceof \OCA\Files_External\Lib\Storage\SMB && extension_loaded('smbclient');
|
||||
}
|
||||
|
||||
public function readAhead(array $ranges, string $spool): ?\Generator {
|
||||
if ($ranges !== [] && $this->s3Reads) {
|
||||
return $this->s3ReadAhead($ranges, $spool);
|
||||
}
|
||||
if ($ranges !== [] && $this->smbNative()) {
|
||||
return $this->smbReadAhead($ranges, $spool);
|
||||
}
|
||||
$reconnect = $this->reconnect;
|
||||
if (!$this->sftpReads || $reconnect === null || $ranges === []) {
|
||||
return null;
|
||||
}
|
||||
// before any connection is opened: an invalid path throws
|
||||
$abs = array_map(fn (array $r): array => [$this->abs($r[0]), $r[1], $r[2]], $ranges);
|
||||
$storages = [];
|
||||
$connections = [];
|
||||
$root = '';
|
||||
$wanted = max(1, min(SftpRangeFetcher::MAX_CONNECTIONS, $this->readAheadConnections));
|
||||
for ($n = 0; $n < $wanted; $n++) {
|
||||
try {
|
||||
$storage = $reconnect();
|
||||
$sftp = SftpReadStream::connection($storage);
|
||||
if ($sftp === null || !is_callable([$storage, 'getRoot'])) {
|
||||
break;
|
||||
}
|
||||
$root = (string)$storage->getRoot();
|
||||
} catch (\Throwable) {
|
||||
break; // the server may allow fewer connections: go on with those that work
|
||||
}
|
||||
$storages[] = $storage;
|
||||
$connections[] = $sftp;
|
||||
}
|
||||
if ($connections === []) {
|
||||
return null;
|
||||
}
|
||||
// in place of a connection that broke: a new storage, kept like the others
|
||||
$connect = static function () use ($reconnect, &$storages): ?object {
|
||||
$storage = $reconnect();
|
||||
$sftp = SftpReadStream::connection($storage);
|
||||
if ($sftp !== null) {
|
||||
$storages[] = $storage;
|
||||
}
|
||||
return $sftp;
|
||||
};
|
||||
return (static function () use (&$storages, $connections, $root, $abs, $spool, $connect): \Generator {
|
||||
// the storages hold their connections: kept until the fetcher is done
|
||||
try {
|
||||
yield from SftpRangeFetcher::fetch($connections, $root, $abs, $spool, $connect);
|
||||
} finally {
|
||||
unset($storages);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
public function getRange(string $path, int $offset, int $length): string {
|
||||
return $this->call(fn () => $this->readRange($path, $offset, $length));
|
||||
}
|
||||
|
||||
private function readRange(string $path, int $offset, int $length): string {
|
||||
try {
|
||||
$chunks = $this->smbChunks($this->abs($path), $offset, $length);
|
||||
if ($chunks !== null) {
|
||||
$data = implode('', iterator_to_array($chunks, false));
|
||||
if (strlen($data) === $length) {
|
||||
return $data;
|
||||
}
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
// read the usual way below, which reports what is wrong, over a new connection
|
||||
$this->smbReconnect();
|
||||
}
|
||||
$fh = $this->get($path);
|
||||
try {
|
||||
if (fseek($fh, $offset) !== 0) {
|
||||
@@ -360,6 +533,218 @@ final class StorageBackend implements IBackend {
|
||||
*
|
||||
* @return list<string>|null
|
||||
*/
|
||||
/**
|
||||
* Bytes per read over SMB (smbChunks()). libsmbclient keeps a large read in flight as many
|
||||
* requests at once; icewind's stream reads 1 MiB at a time. Over a line with a 130 ms round
|
||||
* trip: 4.5 MiB/s in 1 MiB reads, 18 MiB/s in 8 MiB reads (measured 2026-11-04, Samba in a
|
||||
* container behind netem). Without latency it makes no difference.
|
||||
*/
|
||||
public const SMB_READ = 8 * 1048576;
|
||||
|
||||
/**
|
||||
* $length bytes of $abs from $offset over SMB, SMB_READ at a time straight through the native
|
||||
* extension's state of files_external's SMB storage. Null when the storage is not SMB through
|
||||
* the extension, or its internals are not as expected (the usual way reads it then). The
|
||||
* generator throws when a read fails.
|
||||
*
|
||||
* @return \Generator<int, string>|null
|
||||
* @psalm-suppress UndefinedClass icewind/smb comes with files_external, not with this app
|
||||
*/
|
||||
private function smbChunks(string $abs, int $offset, int $length): ?\Generator {
|
||||
$handle = $this->smbHandle($abs);
|
||||
if ($handle === null) {
|
||||
return null;
|
||||
}
|
||||
[$state, $url] = $handle;
|
||||
/** @psalm-suppress MixedMethodCall the state is icewind's NativeState */
|
||||
return (static function () use ($state, $url, $offset, $length): \Generator {
|
||||
$fh = $state->open($url, 'r');
|
||||
try {
|
||||
$state->lseek($fh, $offset, SEEK_SET, $url);
|
||||
for ($got = 0; $got < $length;) {
|
||||
$want = min(self::SMB_READ, $length - $got);
|
||||
$chunk = self::smbRead($state, $fh, $want, $url);
|
||||
if ($chunk === '') {
|
||||
throw new BackendException('Short read in ' . $url);
|
||||
}
|
||||
$got += strlen($chunk);
|
||||
yield $chunk;
|
||||
}
|
||||
} finally {
|
||||
$state->close($fh, $url);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* A whole (small) file over SMB in four round trips: open, read, read up to the end, close.
|
||||
* files_external's fopen() first checks that the file exists (a stat with its attributes) and
|
||||
* icewind's stream reads 1 MiB at a time: about 7 round trips, 1 s per index file from Canada
|
||||
* to the Storage Box (130 ms), so ~8 h for the 27,000 index files of a 1 TB repository. Null
|
||||
* when the storage is not SMB through the extension or anything fails: the usual way then
|
||||
* reads the file and reports a missing one as such.
|
||||
*
|
||||
* @psalm-suppress UndefinedClass icewind/smb comes with files_external, not with this app
|
||||
*/
|
||||
private function smbWhole(string $abs): ?string {
|
||||
$handle = $this->smbHandle($abs);
|
||||
if ($handle === null) {
|
||||
return null;
|
||||
}
|
||||
[$state, $url] = $handle;
|
||||
try {
|
||||
/** @psalm-suppress MixedMethodCall the state is icewind's NativeState */
|
||||
$fh = $state->open($url, 'r');
|
||||
} catch (\Throwable) {
|
||||
return null;
|
||||
}
|
||||
$data = '';
|
||||
try {
|
||||
do {
|
||||
$chunk = self::smbRead($state, $fh, self::SMB_READ, $url);
|
||||
$data .= $chunk;
|
||||
} while ($chunk !== '');
|
||||
} catch (\Throwable) {
|
||||
$data = null;
|
||||
}
|
||||
try {
|
||||
/** @psalm-suppress MixedMethodCall */
|
||||
$state->close($fh, $url);
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
if ($data === null) {
|
||||
$this->smbReconnect(); // a read failed: the usual way reads it over a new connection
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/**
|
||||
* One read of at most $bytes through the native state, checked. When libsmbclient fails a read
|
||||
* (seen when the connection to the server dropped during a long restore), the smbclient
|
||||
* extension does not return false but a string whose length is -1, because it stores
|
||||
* smbc_read()'s -1 in an unsigned length; icewind passes it on. Counting that as data made
|
||||
* the read loop go backwards forever at 100% CPU (DR test 2026-11-05, after 392 GiB).
|
||||
*
|
||||
* @param resource $fh
|
||||
* @psalm-suppress MixedMethodCall, MixedArgument the state is icewind's NativeState
|
||||
*/
|
||||
private static function smbRead(object $state, $fh, int $bytes, string $url): string {
|
||||
$chunk = $state->read($fh, $bytes, $url);
|
||||
$n = is_string($chunk) ? strlen($chunk) : -1;
|
||||
if ($n < 0 || $n > $bytes) {
|
||||
throw new BackendException('Read failed in ' . $url);
|
||||
}
|
||||
return $chunk;
|
||||
}
|
||||
|
||||
/**
|
||||
* After a failed read: drop icewind's connection state, so the next read (the usual way, or
|
||||
* the next range) connects again instead of failing on a connection that is gone.
|
||||
*
|
||||
* @psalm-suppress UndefinedClass icewind/smb comes with files_external, not with this app
|
||||
*/
|
||||
private function smbReconnect(): void {
|
||||
if (!$this->storage instanceof \OCA\Files_External\Lib\Storage\SMB) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
(function (): void {
|
||||
/** @psalm-suppress UndefinedThisPropertyFetch */
|
||||
$share = $this->share;
|
||||
if ($share instanceof \Icewind\SMB\Native\NativeShare) {
|
||||
(function (): void {
|
||||
/** @psalm-suppress UndefinedThisPropertyAssignment */
|
||||
$this->state = null;
|
||||
})->call($share);
|
||||
}
|
||||
})->call($this->storage);
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* icewind's NativeState and the smb:// URL of $abs behind files_external's SMB storage, reached
|
||||
* through bound closures; null when the storage is not SMB through the native extension or its
|
||||
* internals are not as expected.
|
||||
*
|
||||
* @return array{0: object, 1: string}|null
|
||||
* @psalm-suppress UndefinedClass icewind/smb comes with files_external, not with this app
|
||||
*/
|
||||
private function smbHandle(string $abs): ?array {
|
||||
if (!$this->storage instanceof \OCA\Files_External\Lib\Storage\SMB || !extension_loaded('smbclient')) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
/** @var array{0: object, 1: string}|null $handle */
|
||||
$handle = (function (string $path): ?array {
|
||||
/** @psalm-suppress UndefinedThisPropertyFetch */
|
||||
$share = $this->share;
|
||||
if (!$share instanceof \Icewind\SMB\Native\NativeShare) {
|
||||
return null;
|
||||
}
|
||||
/** @psalm-suppress UndefinedMethod */
|
||||
$sharePath = ltrim($this->buildPath($path), '/');
|
||||
/** @psalm-suppress UndefinedMethod */
|
||||
return (fn (string $p): array => [$this->getState(), $this->buildUrl($p)])->call($share, $sharePath);
|
||||
})->call($this->storage, $abs);
|
||||
} catch (\Throwable) {
|
||||
return null;
|
||||
}
|
||||
return $handle;
|
||||
}
|
||||
|
||||
/**
|
||||
* readAhead() over SMB: one range after the other, each in SMB_READ reads into a file in the
|
||||
* spool. Not several at once (PHP cannot wait on more than one libsmbclient call), but whole
|
||||
* ranges of up to ReadAheadPlan::MAX_RANGE in large reads instead of blob by blob in 1 MiB
|
||||
* reads. A range that cannot be read is handed out as null.
|
||||
*
|
||||
* @param list<array{0:string, 1:int, 2:int}> $ranges
|
||||
* @return \Generator<int, resource|null>
|
||||
*/
|
||||
private function smbReadAhead(array $ranges, string $spool): \Generator {
|
||||
// before anything is read: an invalid path throws
|
||||
$abs = array_map(fn (array $r): array => [$this->abs($r[0]), $r[1], $r[2]], $ranges);
|
||||
return (function () use ($abs, $spool): \Generator {
|
||||
$file = rtrim($spool, '/') . '/smb-' . bin2hex(random_bytes(4));
|
||||
try {
|
||||
foreach ($abs as $i => [$path, $offset, $length]) {
|
||||
$fh = null;
|
||||
$out = false;
|
||||
try {
|
||||
$chunks = $this->smbChunks($path, $offset, $length);
|
||||
$out = $chunks === null ? false : fopen($file, 'w+b');
|
||||
if ($chunks !== null && $out !== false) {
|
||||
foreach ($chunks as $chunk) {
|
||||
if (fwrite($out, $chunk) !== strlen($chunk)) {
|
||||
throw new BackendException('Cannot write to the spool');
|
||||
}
|
||||
}
|
||||
$fh = ftell($out) === $length && rewind($out) ? $out : null;
|
||||
if ($fh === null) {
|
||||
fclose($out);
|
||||
}
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
if (is_resource($out)) {
|
||||
fclose($out);
|
||||
}
|
||||
$fh = null; // read the usual way, which reports why, over a new connection
|
||||
$this->smbReconnect();
|
||||
} finally {
|
||||
@unlink($file); // an open stream keeps its data
|
||||
}
|
||||
yield $i => $fh;
|
||||
if (is_resource($fh)) {
|
||||
fclose($fh);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
@unlink($file);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
private function smbNames(string $abs): ?array {
|
||||
if (!$this->storage instanceof \OCA\Files_External\Lib\Storage\SMB) {
|
||||
return null;
|
||||
@@ -429,8 +814,9 @@ final class StorageBackend implements IBackend {
|
||||
$this->lastUsed = microtime(true);
|
||||
}
|
||||
|
||||
/** After a failed request: ask the server (one round trip), as phpseclib may not know it is gone. */
|
||||
private function reopenIfDead(): bool {
|
||||
if ($this->reconnect === null || !$this->sftpReads || $this->alive(false) !== false) {
|
||||
if ($this->reconnect === null || !$this->sftpReads || $this->alive(true) !== false) {
|
||||
return false;
|
||||
}
|
||||
$this->reopen();
|
||||
@@ -459,8 +845,9 @@ final class StorageBackend implements IBackend {
|
||||
if (is_callable([$sftp, 'clearStatCache'])) {
|
||||
$sftp->clearStatCache(); // or stat() answers from memory
|
||||
}
|
||||
$sftp->stat('.');
|
||||
return (bool)$sftp->isConnected();
|
||||
// A Storage Box drops a quiet connection without phpseclib noticing: it still says
|
||||
// connected, and every request fails. So the answer counts, not what it says after.
|
||||
return $sftp->stat('.') !== false && (bool)$sftp->isConnected();
|
||||
} catch (\Throwable) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ final class BackupJob extends TimedJob {
|
||||
private RunMapper $runs,
|
||||
private IAppConfig $appConfig,
|
||||
private LoggerInterface $logger,
|
||||
private \OCA\NgBackup\Service\Schedule $schedule,
|
||||
private \OCA\NgBackup\Service\HeavyJobPause $heavyJobs,
|
||||
) {
|
||||
parent::__construct($time);
|
||||
// Well below the usual 5-minute cron interval: the last run is stamped a moment after cron
|
||||
@@ -42,25 +44,28 @@ final class BackupJob extends TimedJob {
|
||||
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
|
||||
}
|
||||
|
||||
private const MAX_DAILY_ATTEMPTS = 3;
|
||||
|
||||
protected function run($argument): void {
|
||||
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
|
||||
$deadline = microtime(true) + $budget;
|
||||
$this->startScheduled();
|
||||
foreach ($this->runs->findRunning(null, BackupService::KIND_FULL) as $run) {
|
||||
$this->heavyJobs->sync($this->backups->runsInTurn() !== []);
|
||||
// One after the other: runs that read the same disks at the same time are not faster together.
|
||||
foreach ($this->backups->runsInTurn() as $run) {
|
||||
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
|
||||
$next = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
||||
if ($next === $run) {
|
||||
break; // step() hands back the same object when another process holds the run's lease
|
||||
break 2; // step() hands back the same object when another process holds the run's lease
|
||||
}
|
||||
$run = $next;
|
||||
}
|
||||
}
|
||||
$this->failStaleRestores();
|
||||
$this->weeklyPrune();
|
||||
foreach ($this->targets->list() as $t) {
|
||||
$targets = $this->targets->list();
|
||||
foreach ($targets as $t) {
|
||||
$this->alerts->checkStale($t);
|
||||
$othersActive = array_filter($targets, static fn ($o) => $o->getId() !== $t->getId() && (string)$o->getSchedule() !== '') !== [];
|
||||
$this->alerts->checkOff($t, $othersActive);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,25 +106,18 @@ final class BackupJob extends TimedJob {
|
||||
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', $due);
|
||||
}
|
||||
|
||||
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
|
||||
/** Start the backups the schedules of the locations ask for; they run in turn (see LocationSchedule). */
|
||||
private function startScheduled(): void {
|
||||
$at = $this->appConfig->getValueString(Application::APP_ID, 'schedule_time', '');
|
||||
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
|
||||
return;
|
||||
}
|
||||
$todayAt = (int)strtotime(date('Y-m-d') . ' ' . $at);
|
||||
if (time() < $todayAt) {
|
||||
return;
|
||||
}
|
||||
$now = time();
|
||||
foreach ($this->targets->list() as $t) {
|
||||
$last = $this->runs->lastFinished($t->getId(), BackupService::KIND_FULL);
|
||||
if ($this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
|
||||
$moment = \OCA\NgBackup\Service\LocationSchedule::lastMoment((string)$t->getSchedule(), $this->schedule->zoneOr($t->getScheduleZone()), $now);
|
||||
if ($moment === null || $this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== []) {
|
||||
continue;
|
||||
}
|
||||
// After a failure: retry at most MAX_DAILY_ATTEMPTS times a day, an hour apart, instead
|
||||
// of starting (and notifying about) a new full run every five minutes all day.
|
||||
$today = $this->runs->startedSince($t->getId(), BackupService::KIND_FULL, $todayAt);
|
||||
if (count($today) >= self::MAX_DAILY_ATTEMPTS || ($today !== [] && time() - (int)$today[0]->getFinishedAt() < 3600)) {
|
||||
$attempts = $this->runs->startedSince($t->getId(), BackupService::KIND_FULL, $moment);
|
||||
if (!\OCA\NgBackup\Service\LocationSchedule::isDue($moment, (int)$t->getScheduleSince(),
|
||||
$this->dataStarted($this->runs->lastFinished($t->getId(), BackupService::KIND_FULL)),
|
||||
count($attempts), ($attempts[0] ?? null)?->getFinishedAt(), $now)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
@@ -129,4 +127,15 @@ final class BackupJob extends TimedJob {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** When a backup began reading its data: a queued one can start well after it was asked for. */
|
||||
private function dataStarted(?\OCA\NgBackup\Db\Run $run): ?int {
|
||||
if ($run === null) {
|
||||
return null;
|
||||
}
|
||||
$state = json_decode($run->getState(), true);
|
||||
$started = is_array($state) ? ($state['files']['started'] ?? null) : null;
|
||||
$at = is_string($started) ? strtotime($started) : false;
|
||||
return $at !== false ? $at : $run->getStartedAt();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,9 @@ declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Format;
|
||||
use OCA\NgBackup\Service\RestoreService;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\Table;
|
||||
@@ -19,6 +21,7 @@ final class Browse extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private RestoreService $restore,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -43,9 +46,10 @@ final class Browse extends Command {
|
||||
$output->writeln('Nothing here.');
|
||||
return 0;
|
||||
}
|
||||
$zone = $this->schedule->get()['timezone'];
|
||||
$rows = array_map(fn ($i) => [$i['type'] === 'dir' ? $i['name'] . '/' : $i['name'], $this->size($i['size']),
|
||||
$i['type'] === 'dir' ? $i['files'] . ' files' : date('Y-m-d H:i', (int)$i['mtime'])], $items);
|
||||
(new Table($output))->setHeaders(['name', 'size', 'files / modified'])->setRows($rows)->render();
|
||||
$i['type'] === 'dir' ? $i['files'] . ' files' : Format::time((int)$i['mtime'], $zone)], $items);
|
||||
(new Table($output))->setHeaders(['name', 'size', "files / modified ($zone)"])->setRows($rows)->render();
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ declare(strict_types=1);
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\KeyService;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
@@ -18,6 +19,7 @@ use Symfony\Component\Console\Question\Question;
|
||||
final class KeyImport extends Command {
|
||||
public function __construct(
|
||||
private KeyService $keys,
|
||||
private TargetService $targets,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -27,11 +29,19 @@ final class KeyImport extends Command {
|
||||
->setDescription('Disaster recovery: adopt the backup key from a recovery kit on a fresh installation')
|
||||
->addArgument('kit-file', InputArgument::REQUIRED, 'The recovery kit file (occ backup:key:kit on the original installation)')
|
||||
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking')
|
||||
->addOption('replace-unused-key', null, InputOption::VALUE_NONE, 'A new key was made here by mistake: replace it by the kit\'s key, if it was never used (no location, no backup)')
|
||||
->setHelp(<<<'HELP'
|
||||
For a fresh installation only (occ backup:key:init must not have been run here yet). Reads one
|
||||
of the kit's passphrase slots to recover the original master key, so occ backup:target:add opens
|
||||
the existing repository on the location instead of creating a new, empty one.
|
||||
|
||||
A new key was made here by mistake before the location was added? As long as it was never used
|
||||
(no location, no backup), --replace-unused-key replaces it by the kit's key. A key that was used
|
||||
is never replaced.
|
||||
|
||||
No kit? Skip this command: occ backup:target:add then takes the key from the location with one
|
||||
of the old server's passphrases. The kit still helps when the location's config file is damaged.
|
||||
|
||||
Example:
|
||||
occ backup:key:import /root/ng_backup-recovery-kit.json
|
||||
occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=...
|
||||
@@ -40,10 +50,24 @@ HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
if ($this->keys->isInitialized()) {
|
||||
$output->writeln('<error>A backup key already exists here (fingerprint ' . $this->keys->fingerprint() . '). occ backup:key:import is only for a fresh installation.</error>');
|
||||
$replace = (bool)$input->getOption('replace-unused-key');
|
||||
if ($replace && !$this->keys->isInitialized()) {
|
||||
$output->writeln('<error>There is no backup key here to replace: leave out --replace-unused-key.</error>');
|
||||
return 1;
|
||||
}
|
||||
if ($this->keys->isInitialized()) {
|
||||
if (!$this->targets->keyIsUnused()) {
|
||||
$output->writeln('<error>A backup key already exists here (fingerprint ' . $this->keys->fingerprint() . ') and it has been used for a backup location: it is never replaced.</error>');
|
||||
$output->writeln('occ backup:key:import is for a server that is being restored.');
|
||||
return 1;
|
||||
}
|
||||
if (!$replace) {
|
||||
$output->writeln('<error>A backup key already exists here (fingerprint ' . $this->keys->fingerprint() . '), but it was never used: no location, no backup.</error>');
|
||||
$output->writeln('Made by mistake on a server that is being restored? Run this command again with --replace-unused-key:');
|
||||
$output->writeln('the kit\'s key then takes its place, and the passphrases and recovery kit of the unused key stop working.');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
$kitFile = (string)$input->getArgument('kit-file');
|
||||
$raw = @file_get_contents($kitFile);
|
||||
if ($raw === false) {
|
||||
@@ -70,12 +94,16 @@ HELP);
|
||||
$pass = (string)$helper->ask($input, $output, $q);
|
||||
}
|
||||
try {
|
||||
$this->keys->importRecoveryKit($kit, $pass);
|
||||
if ($replace) {
|
||||
$this->targets->importKitReplacingUnusedKey($kit, $pass);
|
||||
} else {
|
||||
$this->keys->importRecoveryKit($kit, $pass);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln('Backup key imported, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
||||
$output->writeln(($replace ? 'The unused key was replaced. ' : '') . 'Backup key imported, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
||||
// Read from the kit file, which is not authenticated beyond the wrapped key: show it so a
|
||||
// wrong or altered value stands out.
|
||||
$output->writeln('Deletion delay for forgotten snapshots: <info>' . $this->keys->deleteDelayDays() . '</info> days (from the kit).');
|
||||
|
||||
@@ -8,6 +8,8 @@ declare(strict_types=1);
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Db\SnapshotMapper;
|
||||
use OCA\NgBackup\Format;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\Table;
|
||||
@@ -19,6 +21,7 @@ final class ListSnapshots extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private SnapshotMapper $snapshots,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -32,18 +35,19 @@ final class ListSnapshots extends Command {
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$name = $input->getArgument('target');
|
||||
$targets = $name === null ? $this->targets->list() : [$this->targets->get($name)];
|
||||
$zone = $this->schedule->get()['timezone'];
|
||||
$rows = [];
|
||||
foreach ($targets as $t) {
|
||||
foreach ($this->snapshots->findForTarget($t->getId()) as $s) {
|
||||
$rows[] = [$t->getName(), $s->getSnapshotId(), date('Y-m-d H:i', $s->getCreatedAt()), $s->getKind(),
|
||||
$s->getFiles(), sprintf('%.1f GiB', ($s->getBytes() ?? 0) / 1073741824), $s->getLabel() ?? ''];
|
||||
$rows[] = [$t->getName(), $s->getSnapshotId(), Format::time($s->getCreatedAt(), $zone), $s->getKind(),
|
||||
$s->getFiles(), sprintf('%.1f GiB', ($s->getBytes() ?? 0) / 1073741824), $s->getPinned() ? 'yes' : '', $s->getLabel() ?? '', $s->getNote() ?? ''];
|
||||
}
|
||||
}
|
||||
if ($rows === []) {
|
||||
$output->writeln('No snapshots yet (occ backup:run).');
|
||||
return 0;
|
||||
}
|
||||
(new Table($output))->setHeaders(['location', 'snapshot', 'date', 'kind', 'files', 'size', 'label'])->setRows($rows)->render();
|
||||
(new Table($output))->setHeaders(['location', 'snapshot', "date ($zone)", 'kind', 'files', 'size', 'pinned', 'label', 'note'])->setRows($rows)->render();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\Table;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class ListUserExports extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:user:list')
|
||||
->setDescription('List the user exports (backup:user:backup) on a location')
|
||||
->addArgument('target', InputArgument::OPTIONAL, 'Location name or id (default: all)')
|
||||
->addArgument('user', InputArgument::OPTIONAL, 'Only the exports of this user id')
|
||||
->setHelp(<<<'HELP'
|
||||
The manifest column is what occ backup:user:restore expects. A safety export made by a restore
|
||||
with --mode=replace is listed here as well.
|
||||
|
||||
Example:
|
||||
occ backup:user:list offsite alice
|
||||
HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$name = $input->getArgument('target');
|
||||
$uid = $input->getArgument('user');
|
||||
$rows = [];
|
||||
try {
|
||||
$targets = $name === null ? $this->targets->list() : [$this->targets->get($name)];
|
||||
foreach ($targets as $t) {
|
||||
foreach ($this->targets->repository($t)->listUserExports($uid) as $path) {
|
||||
$parts = explode('/', $path, 3);
|
||||
$user = $parts[1] ?? '';
|
||||
$exportId = $parts[2] ?? '';
|
||||
$time = \DateTimeImmutable::createFromFormat('Ymd\THis\Z', substr($exportId, 0, 16), new \DateTimeZone('UTC'));
|
||||
$rows[] = [$t->getName(), $user, $time !== false ? $time->format('Y-m-d H:i') : '', $path];
|
||||
}
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
if ($rows === []) {
|
||||
$output->writeln('No user exports yet (occ backup:user:backup).');
|
||||
return 0;
|
||||
}
|
||||
(new Table($output))->setHeaders(['location', 'user', 'date (UTC)', 'manifest'])->setRows($rows)->render();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\PruneService;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class Note extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private PruneService $prune,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:note')
|
||||
->setDescription('Set or remove the note on a snapshot')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
||||
->addArgument('note', InputArgument::OPTIONAL, 'The note, at most ' . PruneService::NOTE_MAX . ' characters; without it the note is removed', '');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$note = (string)$input->getArgument('note');
|
||||
try {
|
||||
$this->prune->setNote($this->targets->get($input->getArgument('target')), $input->getArgument('snapshot'), $note);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln(trim($note) === '' ? 'Note removed.' : 'Note saved.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\HeavyJobPause;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class PauseJobs extends Command {
|
||||
public function __construct(
|
||||
private HeavyJobPause $pause,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:pause-jobs')
|
||||
->setDescription('Show or set whether heavy background jobs (Memories, Recognize, Preview Generator, ...) pause while a backup runs')
|
||||
->addArgument('setting', InputArgument::OPTIONAL, '"on" or "off"; omit to show the current one');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$setting = $input->getArgument('setting');
|
||||
if ($setting !== null) {
|
||||
if ($setting !== 'on' && $setting !== 'off') {
|
||||
$output->writeln('<error>Use "on" or "off".</error>');
|
||||
return 1;
|
||||
}
|
||||
$this->pause->setEnabled($setting === 'on');
|
||||
}
|
||||
$status = $this->pause->status();
|
||||
$output->writeln('Pausing heavy background jobs during a backup: ' . ($status['enabled'] ? 'on' : 'off'));
|
||||
if ($status['jobs'] !== []) {
|
||||
$output->writeln('Paused now: ' . implode(', ', $status['jobs']));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\PruneService;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class Pin extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private PruneService $prune,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:pin')
|
||||
->setDescription('Keep a snapshot: the retention policy does not remove a pinned one')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
||||
->addOption('unpin', null, InputOption::VALUE_NONE, 'Leave the snapshot to the retention policy again');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$unpin = (bool)$input->getOption('unpin');
|
||||
try {
|
||||
$this->prune->setPinned($this->targets->get($input->getArgument('target')), $input->getArgument('snapshot'), !$unpin);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln($unpin ? 'Snapshot unpinned; the retention policy applies to it again.' : 'Snapshot pinned; the retention policy leaves it alone.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ final class Restore extends Command {
|
||||
->addArgument('path', InputArgument::REQUIRED, 'Path in the snapshot (see backup:browse)')
|
||||
->addOption('mode', 'm', InputOption::VALUE_REQUIRED, 'new-folder, merge or replace (user files)', RestoreService::MODE_NEW_FOLDER)
|
||||
->addOption('to-directory', 'd', InputOption::VALUE_REQUIRED, 'Write to this local directory instead (any path, raw files)')
|
||||
->addOption('salvage', null, InputOption::VALUE_NONE, 'Leave out a file that cannot be read or decrypted, list it at the end and go on, instead of stopping')
|
||||
->setHelp(<<<'HELP'
|
||||
User files (data/<user>/files/...) are restored through Nextcloud, so versions, trash and the
|
||||
file cache stay correct. Modes:
|
||||
@@ -42,30 +43,59 @@ Examples:
|
||||
occ backup:restore offsite 537bf7e9... data/alice/files/Documents
|
||||
occ backup:restore offsite 537bf7e9... data/alice/files/Documents/plan.odt --mode=merge
|
||||
occ backup:restore offsite 537bf7e9... config --to-directory=/root/restored-config
|
||||
With --salvage, a file whose data on the location is damaged or missing is left out and listed
|
||||
at the end; everything else is restored. The command then exits with code 2.
|
||||
HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$skipped = $input->getOption('salvage') ? [] : null;
|
||||
try {
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$snapshot = $input->getArgument('snapshot');
|
||||
$path = trim((string)$input->getArgument('path'), '/'); // CLI convenience; the service itself only accepts canonical paths
|
||||
if ($dir = $input->getOption('to-directory')) {
|
||||
$n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir);
|
||||
$n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir, false, null, $skipped);
|
||||
$output->writeln("Restored <info>$n</info> files to $dir.");
|
||||
return 0;
|
||||
return self::reportSkipped($skipped, $output);
|
||||
}
|
||||
$r = $this->restore->restoreUserFiles($target, $snapshot, $path, (string)$input->getOption('mode'),
|
||||
function (int $done, int $total, string $name) use ($output) {
|
||||
if ($output->isVerbose() || $done === $total || $done % 100 === 0) {
|
||||
$output->writeln(" $done/$total $name");
|
||||
}
|
||||
});
|
||||
}, $skipped);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
if ($skipped === null && self::backupLooksDamaged($e)) {
|
||||
$output->writeln('The data of a file on the location is damaged or missing. Run the same command with --salvage to restore everything else and list what could not be restored.');
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
$output->writeln(sprintf('Restored <info>%d</info> files to %s%s.', $r['restored'], $r['target'], $r['trashed'] ? ", moved {$r['trashed']} extra files to the trash" : ''));
|
||||
return 0;
|
||||
return self::reportSkipped($skipped, $output);
|
||||
}
|
||||
|
||||
/** Damaged data, or a file that is gone from the location: what --salvage is for. */
|
||||
public static function backupLooksDamaged(\Throwable $e): bool {
|
||||
return $e instanceof \OCA\NgBackup\Repository\RepositoryException
|
||||
|| $e instanceof \OCA\NgBackup\Crypto\CryptoException
|
||||
|| ($e instanceof \OCA\NgBackup\Backend\BackendException && str_starts_with($e->getMessage(), 'Not found'));
|
||||
}
|
||||
|
||||
/**
|
||||
* The files --salvage left out; exit code 2 when there are any.
|
||||
*
|
||||
* @param array<string, string>|null $skipped
|
||||
*/
|
||||
public static function reportSkipped(?array $skipped, OutputInterface $output): int {
|
||||
if ($skipped === null || $skipped === []) {
|
||||
return 0;
|
||||
}
|
||||
$output->writeln(sprintf('<comment>%d file%s could not be restored (damaged or missing on the location):</comment>', count($skipped), count($skipped) === 1 ? '' : 's'));
|
||||
foreach ($skipped as $path => $reason) {
|
||||
$output->writeln(" $path: $reason");
|
||||
}
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ final class RestoreFull extends Command {
|
||||
private TargetService $targets,
|
||||
private RestoreFullService $restoreFull,
|
||||
private IUserManager $userManager,
|
||||
private \OCP\App\IAppManager $appManager,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -33,14 +34,20 @@ final class RestoreFull extends Command {
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
||||
->addOption('force', null, InputOption::VALUE_NONE, 'Also on an installation that already has more than one user, and without asking')
|
||||
->addOption('install-apps', null, InputOption::VALUE_NONE, 'First install (from the app store) or enable the apps that were enabled at backup time and are missing here')
|
||||
->addOption('skip-missing-tables', null, InputOption::VALUE_NONE, 'Older backups only: leave out tables that do not exist here and whose structure the backup does not record')
|
||||
->addOption('salvage', null, InputOption::VALUE_NONE, 'Restore what can be restored: leave out files that cannot be read, and when the database cannot be restored, put the files in a folder of their own')
|
||||
->setHelp(<<<'HELP'
|
||||
For a FRESH Nextcloud installation only, same major version as the backup, right after
|
||||
occ maintenance:install, occ backup:key:import and occ backup:target:add -- before anyone else
|
||||
uses this instance.
|
||||
|
||||
Every app that was enabled when the snapshot was taken must already be installed here (its
|
||||
database tables must exist before the restore can write into them); the command lists what is
|
||||
missing and refuses to continue rather than guess.
|
||||
Apps: --install-apps first installs (from the app store) or enables every app that was enabled
|
||||
when the snapshot was taken. Apps that are not in the app store (own builds) must be copied into
|
||||
custom_apps by hand; the command lists what is still missing. Tables of apps that are missing or
|
||||
were removed long ago are created from the structure recorded in the backup (backups made with
|
||||
1.0.0-rc.2 or later), so they do not block the restore. Older backups do not record it: the
|
||||
command then lists those tables per app, and --skip-missing-tables leaves them out.
|
||||
|
||||
Steps: the database is replaced (consistent, in one transaction); the data directory is restored
|
||||
directly in place; config.php is merged in -- everything from the backup except the database
|
||||
@@ -49,6 +56,12 @@ here. Other files under config/ (if any) are not restored.
|
||||
|
||||
Afterwards: check trusted_domains and overwritehost/overwriteprotocol match this server, and
|
||||
occ maintenance:repair.
|
||||
When the backup is damaged: --salvage restores what can be restored. Files whose data cannot be
|
||||
read or decrypted are left out and listed. When the database cannot be restored (its dump is
|
||||
damaged, or it does not fit this installation), the database stays as occ maintenance:install
|
||||
made it, and the data directory and config of the backup go to ngb-salvage-<snapshot> in the
|
||||
data directory instead: from there, the files can be put back by hand. The command then exits
|
||||
with code 2.
|
||||
|
||||
Example:
|
||||
occ backup:restore:full offsite 537bf7e9c1a4...
|
||||
@@ -75,12 +88,42 @@ HELP);
|
||||
try {
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$snapshot = (string)$input->getArgument('snapshot');
|
||||
$result = $this->restoreFull->restoreFull($target, $snapshot);
|
||||
$skipMissingTables = (bool)$input->getOption('skip-missing-tables');
|
||||
$salvage = (bool)$input->getOption('salvage');
|
||||
if ($input->getOption('install-apps')) {
|
||||
$check = $this->restoreFull->checkCompatibility($target, $snapshot, $skipMissingTables, $salvage);
|
||||
foreach ($check['missingApps'] as $app) {
|
||||
$this->installApp($app, $output);
|
||||
}
|
||||
}
|
||||
$output->writeln('Nextcloud is in maintenance mode while the restore runs; it is switched off when the restore completes.');
|
||||
$result = $this->restoreFull->restoreFull($target, $snapshot, $skipMissingTables, $salvage);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
$output->writeln('If this happened after the restore had started, the instance may be partly restored: fix the cause and run the same command again with --force.');
|
||||
if (\OCP\Server::get(\OCP\IConfig::class)->getSystemValueBool('maintenance', false)) {
|
||||
$output->writeln('The restore had started: the instance may be partly restored, and maintenance mode stays on so that nobody uses it like that.');
|
||||
$output->writeln('Fix the cause, then switch maintenance mode off and run the same command again with --force, in one line:');
|
||||
$again = 'occ backup:restore:full ' . escapeshellarg((string)$input->getArgument('target')) . ' ' . escapeshellarg((string)$input->getArgument('snapshot'))
|
||||
. ($input->getOption('skip-missing-tables') ? ' --skip-missing-tables' : '') . ($input->getOption('salvage') ? ' --salvage' : '') . ' --force --no-interaction';
|
||||
$output->writeln(' occ maintenance:mode --off && ' . $again);
|
||||
$output->writeln('(Without --install-apps, so the instance is open only for the seconds until the restore starts again. Apps the first run could not install are listed at the end: install those afterwards.)');
|
||||
} else {
|
||||
$output->writeln('If this happened after the restore had started, the instance may be partly restored: fix the cause and run the same command again with --force.');
|
||||
}
|
||||
$output->writeln('If occ then says that Nextcloud needs an upgrade (the database is already the old server\'s), run occ upgrade first.');
|
||||
if (!$input->getOption('salvage') && Restore::backupLooksDamaged($e)) {
|
||||
$output->writeln('The backup itself looks damaged here. --salvage restores everything that can be read and lists the rest.');
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
if ($result['dbError'] !== null) {
|
||||
$output->writeln('<error>Salvage: ' . $result['dbError'] . '</error>');
|
||||
$output->writeln(sprintf('The database stays as occ maintenance:install made it. The data directory and config of the backup (<info>%d</info> data files) are in:', $result['dataFiles']));
|
||||
$output->writeln(' ' . $result['salvageDir']);
|
||||
$output->writeln('Each user\'s files are in data/<user>/files there. Put back what is needed by hand, for example into the files of new users (then occ files:scan).');
|
||||
Restore::reportSkipped($result['skipped'], $output);
|
||||
return 2;
|
||||
}
|
||||
$output->writeln(sprintf(
|
||||
'Database restored (<info>%d</info> tables, <info>%d</info> unchanged), config merged (<info>%d</info> keys), <info>%d</info> data files restored.',
|
||||
count($result['dbReport']['restored']),
|
||||
@@ -88,10 +131,54 @@ HELP);
|
||||
$result['configKeys'],
|
||||
$result['dataFiles'],
|
||||
));
|
||||
if ($result['dbReport']['created'] !== []) {
|
||||
$output->writeln(sprintf('Created <info>%d</info> tables that did not exist here (apps that are missing or were removed): %s',
|
||||
count($result['dbReport']['created']), implode(', ', $result['dbReport']['created'])));
|
||||
}
|
||||
foreach ($result['dbReport']['foreignKeyErrors'] ?? [] as $error) {
|
||||
$output->writeln('<comment>Could not add a foreign key to a created table (the data is restored): ' . $error . '</comment>');
|
||||
}
|
||||
if ($result['dbReport']['missing'] !== []) {
|
||||
$output->writeln('<comment>Left out (structure unknown): ' . implode(', ', $result['dbReport']['missing']) . '</comment>');
|
||||
}
|
||||
if ($result['missingApps'] !== []) {
|
||||
$output->writeln('<comment>These apps were enabled on the old server but are not installed here. Their data is restored; install them so they work again:</comment>');
|
||||
foreach ($result['missingApps'] as $app) {
|
||||
$output->writeln(" occ app:install $app (not in the app store? copy it into custom_apps and run occ app:enable $app)");
|
||||
}
|
||||
}
|
||||
if (!$result['dbConsistent']) {
|
||||
$output->writeln('<comment>Warning: this snapshot\'s database dump was made over several steps (web cron), so it is not one consistent point in time. Check the instance carefully; prefer a snapshot made with system cron.</comment>');
|
||||
}
|
||||
if ($result['maintenanceLeftOn']) {
|
||||
$output->writeln('<comment>Maintenance mode was on before the restore and stays on: switch it off with occ maintenance:mode --off when the instance is ready.</comment>');
|
||||
}
|
||||
$output->writeln('Next: check trusted_domains and overwritehost/overwriteprotocol match this server, then occ maintenance:repair.');
|
||||
return 0;
|
||||
$code = 0;
|
||||
if ($result['configError'] !== null) {
|
||||
$output->writeln('<comment>Salvage: config.php of the backup was not merged (' . $result['configError'] . '). Set at least \'instanceid\', \'passwordsalt\' and \'secret\' to the old server\'s values if you have them, or data encrypted with them (server-side encryption, stored passwords) cannot be read.</comment>');
|
||||
$code = 2;
|
||||
}
|
||||
return max($code, Restore::reportSkipped($result['skipped'], $output));
|
||||
}
|
||||
|
||||
/** Enable an app that is here but switched off, or install it from the app store. */
|
||||
private function installApp(string $app, OutputInterface $output): void {
|
||||
try {
|
||||
$this->appManager->getAppPath($app);
|
||||
$command = 'app:enable'; // here, but switched off
|
||||
} catch (\OCP\App\AppPathNotFoundException) {
|
||||
$command = 'app:install';
|
||||
}
|
||||
$output->writeln("Running occ $command $app ...");
|
||||
try {
|
||||
$code = $this->getApplication()?->find($command)->run(new \Symfony\Component\Console\Input\ArrayInput(['app-id' => $command === 'app:enable' ? [$app] : $app]), $output);
|
||||
} catch (\Throwable $e) {
|
||||
$code = 1;
|
||||
$output->writeln('<comment>' . $e->getMessage() . '</comment>');
|
||||
}
|
||||
if ($code !== 0) {
|
||||
$output->writeln("<comment>Could not $command $app (not in the app store?). Copy it into custom_apps and enable it, or restore without it.</comment>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ final class RestoreUser extends Command {
|
||||
$this->setName('backup:user:restore')
|
||||
->setDescription('Restore a user from a user_migration export (see backup:user:backup)')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('manifest', InputArgument::REQUIRED, 'Manifest path printed by backup:user:backup (users/<uid>/<exportId>)')
|
||||
->addArgument('manifest', InputArgument::REQUIRED, 'Manifest path printed by backup:user:backup (users/<uid>/<exportId>), or a user id with exactly one export')
|
||||
->addOption('mode', 'm', InputOption::VALUE_REQUIRED, 'replace or as-backup', UserRestoreService::MODE_REPLACE)
|
||||
->addOption('force', null, InputOption::VALUE_NONE, 'Replace an existing account without asking (required when not interactive)')
|
||||
->addOption('install-user-migration', null, InputOption::VALUE_NONE, 'Install and enable user_migration if it is missing (non-interactive consent)')
|
||||
@@ -45,16 +45,31 @@ Never imports over a live account. Modes:
|
||||
compare and transfer by hand; Nextcloud cannot rename a user id, so the backup gets
|
||||
the new name, never the existing account
|
||||
|
||||
occ backup:user:list shows the exports on a location. A user id alone works when that user
|
||||
has exactly one export there; with more, the command lists them and you pass one.
|
||||
|
||||
Example:
|
||||
occ backup:user:restore offsite users/alice/20261005T120000Z-ab12cd34
|
||||
occ backup:user:restore offsite alice
|
||||
occ backup:user:restore offsite users/alice/20261005T120000Z-ab12cd34 --mode=as-backup
|
||||
HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$mode = (string)$input->getOption('mode');
|
||||
$uid = explode('/', (string)$input->getArgument('manifest'))[1] ?? '';
|
||||
if ($mode === UserRestoreService::MODE_REPLACE && $uid !== '' && $this->userManager->userExists($uid) && !$input->getOption('force')) {
|
||||
try {
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$ref = (string)$input->getArgument('manifest');
|
||||
$manifest = $this->targets->repository($target)->resolveUserExport($ref);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
if ($manifest !== $ref) {
|
||||
$output->writeln("Using $manifest.");
|
||||
}
|
||||
$uid = explode('/', $manifest)[1];
|
||||
if ($mode === UserRestoreService::MODE_REPLACE && $this->userManager->userExists($uid) && !$input->getOption('force')) {
|
||||
// Replace deletes the live account (after a safety export). Shares to others and data of
|
||||
// apps without a user_migration migrator do not come back, so never do it unasked.
|
||||
if (!$input->isInteractive()) {
|
||||
@@ -69,9 +84,8 @@ HELP);
|
||||
}
|
||||
try {
|
||||
$output->writeln('<comment>Note: on restore, user_migration only imports the app settings on its own allowlist (e.g. calendar view and reminder settings); other app settings of this user are skipped. Reason: NG Backup uses user_migration\'s importer; widening the list would mean building a settings importer of our own.</comment>');
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$this->installer->ensureEnabled($input, $output);
|
||||
$user = $this->userRestore->restoreUser($target, (string)$input->getArgument('manifest'), $mode);
|
||||
$user = $this->userRestore->restoreUser($target, $manifest, $mode);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
|
||||
@@ -79,6 +79,9 @@ HELP);
|
||||
if (($f['unreadable'] ?? 0) > 0) {
|
||||
$output->writeln(sprintf('<comment> warning: %d unreadable files or folders are missing from this backup (see the server log)</comment>', $f['unreadable']));
|
||||
}
|
||||
if (($f['badNames'] ?? 0) > 0) {
|
||||
$output->writeln(sprintf('<comment> warning: %d files or folders are missing from this backup because their name is not supported; rename them and the next backup includes them (see the server log)</comment>', $f['badNames']));
|
||||
}
|
||||
if (($f['changing'] ?? 0) > 0) {
|
||||
$output->writeln(sprintf('<comment> warning: %d files kept changing and were saved as read (see the server log)</comment>', $f['changing']));
|
||||
}
|
||||
|
||||
@@ -8,7 +8,10 @@ declare(strict_types=1);
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Db\RunMapper;
|
||||
use OCA\NgBackup\Format;
|
||||
use OCA\NgBackup\Service\CronHealth;
|
||||
use OCA\NgBackup\Service\KeyService;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use OCA\NgBackup\Service\VerifyService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
@@ -26,6 +29,9 @@ final class Status extends Command {
|
||||
private TargetService $targets,
|
||||
private RunMapper $runs,
|
||||
private VerifyService $verify,
|
||||
private Schedule $schedule,
|
||||
private CronHealth $cron,
|
||||
private \OCA\NgBackup\Service\BackupService $backups,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -36,12 +42,17 @@ final class Status extends Command {
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$zone = $this->schedule->get()['timezone'];
|
||||
$cronProblem = CronHealth::describe($this->cron->check());
|
||||
if ($cronProblem !== null) {
|
||||
$output->writeln('<error>' . $cronProblem . '</error>');
|
||||
}
|
||||
if (!$this->keys->isInitialized()) {
|
||||
$output->writeln('Key: <error>not created</error> (occ backup:key:init)');
|
||||
} else {
|
||||
$c = $this->keys->recoveryKitConfirmation();
|
||||
$output->writeln('Key: ' . $this->keys->fingerprint() . ', recovery kit '
|
||||
. ($c ? 'confirmed by ' . $c['uid'] . ' on ' . date('Y-m-d H:i', $c['time']) : '<error>not confirmed</error> (occ backup:key:confirm)'));
|
||||
. ($c ? 'confirmed by ' . $c['uid'] . ' on ' . Format::time($c['time'], $zone) . " ($zone)" : '<error>not confirmed</error> (occ backup:key:confirm)'));
|
||||
}
|
||||
$names = [];
|
||||
foreach ($this->targets->list() as $t) {
|
||||
@@ -51,15 +62,17 @@ final class Status extends Command {
|
||||
$rows = [];
|
||||
$all = (bool)$input->getOption('all');
|
||||
$shown = $this->runs->findShown($all ? null : time() - self::RECENT_DAYS * 86400);
|
||||
$queued = array_map(static fn (\OCA\NgBackup\Db\Run $r): int => $r->getId(), array_slice($this->backups->runsInTurn(), 1));
|
||||
foreach ($shown as $r) {
|
||||
$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
|
||||
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . self::percent($r) . ')' : ''),
|
||||
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? \OCA\NgBackup\Format::duration($r->getFinishedAt() - $r->getStartedAt()) : '',
|
||||
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(),
|
||||
in_array($r->getId(), $queued, true) ? 'queued' : ($r->getStatus() === 'running' ? self::runningStatus($r) : $r->getStatus()),
|
||||
Format::time($r->getStartedAt(), $zone), $r->getFinishedAt() ? Format::duration($r->getFinishedAt() - $r->getStartedAt()) : '',
|
||||
$s['files'] ?? '', isset($s['bytes']) ? self::size((int)$s['bytes']) : '',
|
||||
isset($s['uploaded']) ? self::size((int)$s['uploaded']) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
|
||||
}
|
||||
if ($rows !== []) {
|
||||
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'size', 'uploaded', 'error'])->setRows($rows)->render();
|
||||
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', "started ($zone)", 'took', 'files', 'size', 'uploaded', 'error'])->setRows($rows)->render();
|
||||
}
|
||||
$older = $all ? 0 : $this->runs->countAll() - count($shown);
|
||||
if ($older > 0) {
|
||||
@@ -84,8 +97,24 @@ final class Status extends Command {
|
||||
return sprintf('%.1f TiB', $b);
|
||||
}
|
||||
|
||||
/** " 47/183 tables" in the database phase, " 68%" or " ~68%" in the files phase; empty without a figure */
|
||||
private static function percent(\OCA\NgBackup\Db\Run $r): string {
|
||||
$p = \OCA\NgBackup\Job\BackupRun::progressOf($r);
|
||||
return $p === null ? '' : sprintf(' %s%d%%', $p['approximate'] ? '~' : '', (int)floor($p['fraction'] * 100.0));
|
||||
if ($p === null) {
|
||||
return '';
|
||||
}
|
||||
if (isset($p['tables'], $p['totalTables'])) {
|
||||
return sprintf(': %d/%d tables', $p['tables'], $p['totalTables']);
|
||||
}
|
||||
return sprintf(' %s%d%%', $p['approximate'] ? '~' : '', (int)floor($p['fraction'] * 100.0));
|
||||
}
|
||||
|
||||
/** "running (db: 47/183 tables)", or "waiting for background jobs (last step 12 min ago)" */
|
||||
private static function runningStatus(\OCA\NgBackup\Db\Run $r): string {
|
||||
$waiting = $r->getKind() === \OCA\NgBackup\Service\BackupService::KIND_FULL ? CronHealth::waitingFor($r) : null;
|
||||
if ($waiting !== null) {
|
||||
return 'waiting for background jobs (' . $r->getPhase() . self::percent($r) . ', last step ' . CronHealth::ago($waiting) . ')';
|
||||
}
|
||||
return 'running (' . $r->getPhase() . self::percent($r) . ')';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,9 @@ use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Helper\QuestionHelper;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Question\Question;
|
||||
|
||||
final class TargetAdd extends Command {
|
||||
public function __construct(
|
||||
@@ -31,10 +33,25 @@ final class TargetAdd extends Command {
|
||||
->addOption('auth', 'a', InputOption::VALUE_REQUIRED, 'Auth mechanism id (see backup:target:backends)')
|
||||
->addOption('option', 'o', InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Backend/auth option key=value (repeatable)')
|
||||
->addOption('path', 'p', InputOption::VALUE_REQUIRED, 'Folder on the location for this repository', 'ng_backup')
|
||||
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Disaster recovery without the recovery kit: read the passphrase from this file instead of asking')
|
||||
->addOption('repair-config', null, InputOption::VALUE_NONE, 'The location\'s config file is gone: write it again with the key of this server or of the imported recovery kit (locations backed up by 1.1.0 or newer)')
|
||||
->addOption('replace-unused-key', null, InputOption::VALUE_NONE, 'Disaster recovery after a new key was made here by mistake: replace that key, if it was never used, by the key of the backups at this location (asks for a passphrase of the old server)')
|
||||
->setHelp(<<<'HELP'
|
||||
The location is reached with files_external's own connection code, but it is never added as an
|
||||
external storage: it does not appear in Files for anyone. Credentials are stored encrypted.
|
||||
|
||||
Disaster recovery without the recovery kit: on a server without a backup key, a location that
|
||||
holds backups gives its key with one of the old server's passphrases. Run this command first
|
||||
(no occ backup:key:import), then occ backup:list and occ backup:restore:full.
|
||||
|
||||
A new key was made here by mistake (occ backup:key:init or the web page) before the location
|
||||
was added? As long as that key was never used (no location, no backup), --replace-unused-key
|
||||
replaces it by the key of the backups. A key that was used is never replaced.
|
||||
|
||||
With the recovery kit: occ backup:key:import first; this command then asks for nothing more.
|
||||
If the location's config file is gone, only the kit has the key: import it, then add the
|
||||
location with --repair-config.
|
||||
|
||||
Examples:
|
||||
occ backup:target:add offsite amazons3 -a amazons3::accesskey -o bucket=backups -o hostname=s3.example.com -o region=eu-central-1 -o use_ssl=true -o use_path_style=true -o key=AKIA... -o secret=... --path=nextcloud
|
||||
|
||||
@@ -43,10 +60,6 @@ HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
if (!$this->keys->isInitialized()) {
|
||||
$output->writeln('<error>No backup key yet: run occ backup:key:init first.</error>');
|
||||
return 1;
|
||||
}
|
||||
$options = [];
|
||||
foreach ($input->getOption('option') as $pair) {
|
||||
if (!str_contains($pair, '=')) {
|
||||
@@ -60,14 +73,167 @@ HELP);
|
||||
$output->writeln('<error>--auth is required (see occ backup:target:backends)</error>');
|
||||
return 1;
|
||||
}
|
||||
$name = (string)$input->getArgument('name');
|
||||
$backend = (string)$input->getArgument('backend');
|
||||
$auth = (string)$input->getOption('auth');
|
||||
$path = (string)$input->getOption('path');
|
||||
$passphrase = null;
|
||||
$repair = (bool)$input->getOption('repair-config');
|
||||
$replace = (bool)$input->getOption('replace-unused-key');
|
||||
if ($replace) {
|
||||
if ($repair) {
|
||||
$output->writeln('<error>--replace-unused-key needs the config at the location; with the config gone only the kit helps: occ backup:key:import <kit file> --replace-unused-key, then --repair-config.</error>');
|
||||
return 1;
|
||||
}
|
||||
if (!$this->keys->isInitialized()) {
|
||||
$output->writeln('<error>There is no backup key here to replace: leave out --replace-unused-key.</error>');
|
||||
return 1;
|
||||
}
|
||||
if (!$this->targets->keyIsUnused()) {
|
||||
$output->writeln('<error>The backup key of this server (fingerprint ' . $this->keys->fingerprint() . ') has been used for a backup location: it is never replaced.</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln('The backup key of this server (fingerprint ' . $this->keys->fingerprint() . ') was never used. It is replaced by the key of the backups at');
|
||||
$output->writeln('this location, once a passphrase of the old server opens it. Its own passphrases and recovery kit then stop working.');
|
||||
$passphrase = $this->passphrase($input, $output);
|
||||
if ($passphrase === null) {
|
||||
return 1;
|
||||
}
|
||||
} elseif (!$this->keys->isInitialized()) {
|
||||
try {
|
||||
$state = $this->targets->locationState($backend, $auth, $options, $path);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
if ($state === TargetService::LOCATION_EMPTY) {
|
||||
$output->writeln('<error>No backup key yet, and no backups at this location.</error>');
|
||||
$output->writeln('Restoring a server (disaster recovery)? Check the connection details and --path: they must be the same as on the old server.');
|
||||
$output->writeln('Only for a new location: create a new key first with occ backup:key:init. A new key cannot read existing backups.');
|
||||
return 1;
|
||||
}
|
||||
if ($state === TargetService::LOCATION_DATA_WITHOUT_CONFIG) {
|
||||
$output->writeln('<error>This location holds backups, but its config file is gone, and the key was stored in it.</error>');
|
||||
$output->writeln('Now only the recovery kit has the key. Import it, then run this command again with --repair-config:');
|
||||
$output->writeln(' occ backup:key:import /path/to/ng_backup-kit.json');
|
||||
$output->writeln('Do not run occ backup:key:init: a new key cannot read these backups.');
|
||||
return 1;
|
||||
}
|
||||
if ($repair) {
|
||||
$output->writeln('<error>The config at this location is there: leave out --repair-config.</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln('No backup key on this server yet, and this location holds backups. Their key is stored there too,');
|
||||
$output->writeln('locked with the passphrases of the old server: one of them unlocks it (the recovery kit is not needed).');
|
||||
$passphrase = $this->passphrase($input, $output);
|
||||
if ($passphrase === null) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
// Adopting an existing repository reads its whole index first, which takes a while on a
|
||||
// large backup or a slow line; show how far it is instead of looking stuck.
|
||||
$output->writeln('Connecting to the location; with existing backups this reads their index first ...');
|
||||
try {
|
||||
$r = $this->targets->add($input->getArgument('name'), $input->getArgument('backend'), $input->getOption('auth'), $options, (string)$input->getOption('path'));
|
||||
} catch (\Throwable $e) {
|
||||
$r = $passphrase === null
|
||||
? $this->targets->add($name, $backend, $auth, $options, $path, self::progress($output), $repair)
|
||||
: $this->targets->addWithPassphrase($name, $backend, $auth, $options, $path, $passphrase, self::progress($output), $replace);
|
||||
} catch (\OCA\NgBackup\Crypto\CryptoException $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
if (!$this->keys->isInitialized() || $replace) {
|
||||
$output->writeln('Nothing was changed. Run the command again to try another passphrase, or use the recovery kit (occ backup:key:import).');
|
||||
}
|
||||
return 1;
|
||||
} catch (\Throwable $e) {
|
||||
if ($output->isDecorated()) {
|
||||
$output->write("\r\033[K"); // clear an unfinished progress line
|
||||
}
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
if (str_contains($e->getMessage(), 'different key') && !$replace) {
|
||||
$output->writeln('The backup key here (fingerprint ' . $this->keys->fingerprint() . ') is not the one these backups were made with.');
|
||||
$output->writeln('Restoring a server? The recovery kit must be the old server\'s: its fingerprint is in the kit file. Nothing was changed.');
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
if ($repair) {
|
||||
$output->writeln('The config of the location was written again from the recovery kit.');
|
||||
}
|
||||
$output->writeln(sprintf('Location <info>%s</info> added (id %d), repository %s %s.', $r['target']->getName(), $r['target']->getId(),
|
||||
$r['target']->getRepositoryId(), $r['created'] ? 'created' : 'found and adopted'));
|
||||
if ($passphrase !== null) {
|
||||
$output->writeln(($replace ? 'The unused key was replaced. ' : '') . 'Backup key taken from the location, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
||||
$days = 'Deletion delay for forgotten snapshots: <info>' . $this->keys->deleteDelayDays() . '</info> days';
|
||||
$output->writeln(match (true) {
|
||||
($r['deleteDelayDays'] ?? null) !== null => $days . ' (from the location).',
|
||||
$r['settingsUnreadable'] ?? false => '<comment>' . $days . ' (the longest: the settings on the location could not be read or hold no valid delay, so nothing leaves the Bin sooner than on the old server; removed restore points take space for that long).</comment>',
|
||||
default => $days . ' (the default: the location has none stored, its backups were made before NG Backup 1.1.0).',
|
||||
});
|
||||
$output->writeln('Next: <info>occ backup:list</info> and <info>occ backup:restore:full</info>. Before backups run again, download a new recovery kit and confirm it (<info>occ backup:key:kit</info>, <info>occ backup:key:confirm</info>).');
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private function passphrase(InputInterface $input, OutputInterface $output): ?string {
|
||||
$file = $input->getOption('passphrase-file');
|
||||
if ($file !== null) {
|
||||
$raw = @file_get_contents((string)$file);
|
||||
if ($raw === false) {
|
||||
$output->writeln('<error>Cannot read the passphrase file</error>');
|
||||
return null;
|
||||
}
|
||||
return rtrim($raw, "\r\n");
|
||||
}
|
||||
if (!$input->isInteractive()) {
|
||||
$output->writeln('<error>Give the passphrase with --passphrase-file, or run the command without --no-interaction to type it.</error>');
|
||||
return null;
|
||||
}
|
||||
$q = (new Question('Passphrase of the old server (one of its slots): '))->setHidden(true)->setHiddenFallback(false);
|
||||
/** @var QuestionHelper $helper */
|
||||
$helper = $this->getHelper('question');
|
||||
return (string)$helper->ask($input, $output, $q);
|
||||
}
|
||||
|
||||
/**
|
||||
* Progress of reading the index: on a terminal one line that is updated in place, otherwise
|
||||
* (a log, a pipe) a line every 30 seconds. The time left is estimated from the rate so far.
|
||||
*
|
||||
* @return callable(int, int): void
|
||||
*/
|
||||
private static function progress(OutputInterface $output): callable {
|
||||
$shown = 0.0;
|
||||
$first = null; // [time, files done] when the first files were in
|
||||
$inPlace = $output->isDecorated();
|
||||
return static function (int $done, int $total) use ($output, &$shown, &$first, $inPlace): void {
|
||||
$now = microtime(true);
|
||||
if ($first === null && $done > 0) {
|
||||
$first = [$now, $done];
|
||||
}
|
||||
if ($done < $total && $now - $shown < ($inPlace ? 1 : 30)) {
|
||||
return;
|
||||
}
|
||||
$shown = $now;
|
||||
$line = sprintf('Reading the index: %s/%s files (%d%%)', number_format($done), number_format($total), intdiv(100 * $done, max(1, $total)));
|
||||
$left = $first === null ? null : self::minutesLeft($now - $first[0], $done - $first[1], $total - $done, $total);
|
||||
if ($left !== null) {
|
||||
$line .= $left <= 1 ? ', less than a minute left' : sprintf(', about %d min left', $left);
|
||||
}
|
||||
if ($inPlace) {
|
||||
$output->write("\r\033[K" . $line . ($done >= $total ? "\n" : ''));
|
||||
} else {
|
||||
$output->writeln($line);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Minutes left for reading the index, from the rate since the first files were in: the first
|
||||
* file also carries the time to connect, which made an S3 location with 4,568 index files
|
||||
* say "about 642 min left" for something done within a minute. No estimate until 5% of the
|
||||
* files, or 50 files, came in after that, and at least 5 seconds passed.
|
||||
*/
|
||||
public static function minutesLeft(float $elapsed, int $done, int $left, int $total): ?int {
|
||||
if ($left <= 0 || $done <= 0 || $elapsed < 5.0 || ($done < 50 && $done * 20 < $total)) {
|
||||
return null;
|
||||
}
|
||||
return (int)ceil($elapsed / (float)$done * (float)$left / 60.0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,8 @@ final class TargetList extends Base {
|
||||
$rows = [];
|
||||
foreach ($this->targets->list() as $t) {
|
||||
$row = ['id' => $t->getId(), 'name' => $t->getName(), 'backend' => $t->getBackend(), 'path' => $t->getBasePath(),
|
||||
'repository' => $t->getRepositoryId(), 'added' => date('Y-m-d H:i', $t->getCreatedAt())];
|
||||
'repository' => $t->getRepositoryId(), 'added' => date('Y-m-d H:i', $t->getCreatedAt()),
|
||||
'schedule' => (string)$t->getSchedule() !== '' ? $t->getSchedule() : 'off'];
|
||||
if ($input->getOption('test')) {
|
||||
try {
|
||||
$repo = $this->targets->repository($t);
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class TargetSchedule extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:target:schedule')
|
||||
->setDescription('Show or set when a location is backed up')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('schedule', InputArgument::OPTIONAL, '"off", HH:MM for once a day, or HH:MM/N for every N hours from HH:MM; omit to show the current one')
|
||||
->addOption('timezone', null, InputOption::VALUE_REQUIRED, 'Time zone of the time, e.g. Europe/Amsterdam');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$t = $this->targets->get($input->getArgument('target'));
|
||||
$schedule = $input->getArgument('schedule');
|
||||
if ($schedule === null && $input->getOption('timezone') !== null) {
|
||||
$output->writeln('<error>--timezone goes with a schedule, e.g. backup:target:schedule ' . $t->getName() . ' 02:30 --timezone=Europe/Amsterdam</error>');
|
||||
return 1;
|
||||
}
|
||||
$zone = $input->getOption('timezone') ?? $this->schedule->zoneOr($t->getScheduleZone());
|
||||
if ($schedule !== null) {
|
||||
try {
|
||||
$t = $this->targets->setSchedule($t, $schedule, (string)$zone);
|
||||
} catch (\InvalidArgumentException $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
$current = (string)$t->getSchedule();
|
||||
$output->writeln($t->getName() . ': ' . ($current === '' ? 'off' : $current . ' (' . $zone . ')'));
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,9 @@ declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Format;
|
||||
use OCA\NgBackup\Service\KeyService;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\Table;
|
||||
@@ -19,6 +21,7 @@ final class Trash extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private KeyService $keys,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -32,11 +35,14 @@ final class Trash extends Command {
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$name = $input->getArgument('target');
|
||||
$delay = $this->keys->deleteDelayDays() * 86400;
|
||||
$zone = $this->schedule->get()['timezone'];
|
||||
$rows = [];
|
||||
foreach ($name === null ? $this->targets->list() : [$this->targets->get($name)] as $t) {
|
||||
foreach ($this->targets->repository($t)->trash() as $id => $info) {
|
||||
$rows[] = [$t->getName(), $id, (string)($info['time'] ?? ''), $info['label'], date('Y-m-d H:i', $info['forgottenAt']), $info['by'],
|
||||
date('Y-m-d H:i', $info['forgottenAt'] + $delay)];
|
||||
$time = $info['time'] ?? null;
|
||||
$made = is_string($time) ? strtotime($time) : false;
|
||||
$rows[] = [$t->getName(), $id, $made !== false ? Format::time($made, $zone) : '', $info['label'], Format::time($info['forgottenAt'], $zone), $info['by'],
|
||||
Format::time($info['forgottenAt'] + $delay, $zone)];
|
||||
}
|
||||
}
|
||||
if ($rows === []) {
|
||||
@@ -44,6 +50,7 @@ final class Trash extends Command {
|
||||
return 0;
|
||||
}
|
||||
(new Table($output))->setHeaders(['location', 'snapshot', 'made', 'label', 'removed', 'by', 'deleted after'])->setRows($rows)->render();
|
||||
$output->writeln("Times in $zone. The retention policy keeps the newest snapshots and the last of each recent day, week and month; it moves the others here, and they are deleted permanently after " . $this->keys->deleteDelayDays() . ' days.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +47,9 @@ final class AdminApiController extends Controller {
|
||||
private \OCA\NgBackup\Service\AlertService $alerts,
|
||||
private \OCA\NgBackup\Service\KeySlotService $slots,
|
||||
private \OCA\NgBackup\Service\PruneService $prune,
|
||||
private \OCA\NgBackup\Service\Schedule $schedule,
|
||||
private \OCA\NgBackup\Service\HeavyJobPause $heavyJobs,
|
||||
private \OCA\NgBackup\Service\CronHealth $cron,
|
||||
) {
|
||||
parent::__construct(Application::APP_ID, $request);
|
||||
}
|
||||
@@ -57,10 +60,12 @@ final class AdminApiController extends Controller {
|
||||
foreach ($this->targets->list() as $t) {
|
||||
$targets[] = ['id' => $t->getId(), 'name' => $t->getName(), 'backend' => $t->getBackend(), 'path' => $t->getBasePath(),
|
||||
'repositoryId' => $t->getRepositoryId(), 'createdAt' => $t->getCreatedAt(), 'appendOnly' => (bool)$t->getAppendOnly(),
|
||||
'maxBytes' => $t->getMaxBytes(), 'storedBytes' => $this->alerts->lastStoredBytes($t)];
|
||||
'maxBytes' => $t->getMaxBytes(), 'storedBytes' => $this->alerts->lastStoredBytes($t),
|
||||
'schedule' => (string)$t->getSchedule(), 'scheduleZone' => $this->schedule->zoneOr($t->getScheduleZone())];
|
||||
}
|
||||
$shown = $this->runs->findShown($allRuns ? null : time() - \OCA\NgBackup\Command\Status::RECENT_DAYS * 86400);
|
||||
$runs = array_map(fn (Run $r) => $this->runJson($r), $shown);
|
||||
$queued = array_flip(array_map(static fn (Run $r): int => $r->getId(), array_slice($this->backups->runsInTurn(), 1)));
|
||||
$runs = array_map(fn (Run $r) => $this->runJson($r) + ['queued' => isset($queued[$r->getId()])], $shown);
|
||||
return new JSONResponse([
|
||||
'key' => ['initialized' => $this->keys->isInitialized(), 'fingerprint' => $this->keys->fingerprint(),
|
||||
'confirmation' => $this->keys->recoveryKitConfirmation(), 'everConfirmed' => $this->keys->isInitialized() && $this->keys->everConfirmed(),
|
||||
@@ -72,10 +77,48 @@ final class AdminApiController extends Controller {
|
||||
'runs' => $runs,
|
||||
'olderRuns' => $allRuns ? 0 : $this->runs->countAll() - count($shown),
|
||||
'recentDays' => \OCA\NgBackup\Command\Status::RECENT_DAYS,
|
||||
'schedule' => $this->appConfig->getValueString(Application::APP_ID, 'schedule_time', ''),
|
||||
'cron' => $this->cron->check(),
|
||||
'heavyJobs' => $this->heavyJobs->status(),
|
||||
]);
|
||||
}
|
||||
|
||||
public function setPauseHeavyJobs(bool $enabled): JSONResponse {
|
||||
$this->heavyJobs->setEnabled($enabled);
|
||||
return new JSONResponse(['enabled' => $enabled]);
|
||||
}
|
||||
|
||||
/** Seconds of backup work one tick() does: well within a normal web request. */
|
||||
public const TICK_SECONDS = 10.0;
|
||||
/** Seconds one request for the Bin reads, before the page shows how far it is and asks for more. */
|
||||
private const TRASH_PAGE_SECONDS = 5.0;
|
||||
|
||||
/**
|
||||
* Move running backups forward from the open admin page, when background jobs do not run on a
|
||||
* working system cron (#13). The page calls this while a backup runs, so "Back up now" works
|
||||
* without cron as long as the page stays open; scheduled backups still need cron. With a
|
||||
* working system cron this does nothing: the background job does the work.
|
||||
*/
|
||||
public function tick(): JSONResponse {
|
||||
if ($this->cron->check()['ok']) {
|
||||
return new JSONResponse(['stepped' => 0]);
|
||||
}
|
||||
$deadline = microtime(true) + self::TICK_SECONDS;
|
||||
$stepped = 0;
|
||||
$this->heavyJobs->sync($this->backups->runsInTurn() !== []);
|
||||
foreach ($this->backups->runsInTurn() as $run) {
|
||||
if (microtime(true) >= $deadline) {
|
||||
break;
|
||||
}
|
||||
// step() returns at once when the background job (or another tab) holds the run.
|
||||
$next = $this->backups->step($run, $deadline);
|
||||
$stepped++;
|
||||
if ($next->getStatus() === Run::RUNNING) {
|
||||
break; // not finished: the next one waits for its turn
|
||||
}
|
||||
}
|
||||
return new JSONResponse(['stepped' => $stepped]);
|
||||
}
|
||||
|
||||
#[PasswordConfirmationRequired]
|
||||
public function initKey(string $passphrase, int $deleteDelayDays = \OCA\NgBackup\Service\KeyService::DEFAULT_DELETE_DELAY, string $label = 'Slot 1'): JSONResponse {
|
||||
try {
|
||||
@@ -144,15 +187,18 @@ final class AdminApiController extends Controller {
|
||||
}
|
||||
}
|
||||
|
||||
public function trash(int $targetId): JSONResponse {
|
||||
public function trash(int $targetId, string $after = ''): JSONResponse {
|
||||
try {
|
||||
$delay = $this->keys->deleteDelayDays() * 86400;
|
||||
$repo = $this->targets->repository($this->targets->get((string)$targetId));
|
||||
// In parts, so that the page can show how far it is (a read per removed restore point).
|
||||
$page = $repo->trashPage($after, microtime(true) + self::TRASH_PAGE_SECONDS);
|
||||
$out = [];
|
||||
foreach ($this->targets->repository($this->targets->get((string)$targetId))->trash() as $id => $info) {
|
||||
foreach ($page['items'] as $id => $info) {
|
||||
$out[] = ['id' => $id, 'label' => $info['label'], 'time' => $info['time'], 'removedAt' => $info['forgottenAt'],
|
||||
'by' => $info['by'], 'deleteAfter' => $info['forgottenAt'] + $delay];
|
||||
}
|
||||
return new JSONResponse($out);
|
||||
return new JSONResponse(['total' => $page['total'], 'items' => $out]);
|
||||
} catch (\Throwable $e) {
|
||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||
}
|
||||
@@ -240,23 +286,43 @@ final class AdminApiController extends Controller {
|
||||
return new JSONResponse($this->runJson($run));
|
||||
}
|
||||
|
||||
public function setSchedule(string $time): JSONResponse {
|
||||
if ($time !== '' && !preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $time)) {
|
||||
return new JSONResponse(['error' => 'Use HH:MM or empty'], Http::STATUS_BAD_REQUEST);
|
||||
/** @param string $timezone the administrator's zone (the browser's); empty: the default one */
|
||||
public function setTargetSchedule(int $id, string $schedule, string $timezone = ''): JSONResponse {
|
||||
try {
|
||||
$t = $this->targets->setSchedule($this->targets->get((string)$id), $schedule, $timezone !== '' ? $timezone : $this->schedule->get()['timezone']);
|
||||
} catch (\Throwable $e) {
|
||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||
}
|
||||
$this->appConfig->setValueString(Application::APP_ID, 'schedule_time', $time);
|
||||
return new JSONResponse(['schedule' => $time]);
|
||||
return new JSONResponse(['schedule' => $t->getSchedule(), 'scheduleZone' => $t->getScheduleZone()]);
|
||||
}
|
||||
|
||||
public function snapshots(int $targetId): JSONResponse {
|
||||
$out = [];
|
||||
foreach ($this->snapshots->findForTarget($targetId) as $s) {
|
||||
$out[] = ['id' => $s->getSnapshotId(), 'kind' => $s->getKind(), 'label' => $s->getLabel(), 'createdAt' => $s->getCreatedAt(),
|
||||
'files' => $s->getFiles(), 'bytes' => $s->getBytes()];
|
||||
'files' => $s->getFiles(), 'bytes' => $s->getBytes(), 'note' => $s->getNote(), 'pinned' => (bool)$s->getPinned()];
|
||||
}
|
||||
return new JSONResponse($out);
|
||||
}
|
||||
|
||||
public function setSnapshotNote(int $targetId, string $snapshotId, string $note = ''): JSONResponse {
|
||||
try {
|
||||
$this->prune->setNote($this->targets->get((string)$targetId), $snapshotId, $note);
|
||||
} catch (\Throwable $e) {
|
||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||
}
|
||||
return new JSONResponse(['ok' => true]);
|
||||
}
|
||||
|
||||
public function setSnapshotPinned(int $targetId, string $snapshotId, bool $pinned = true): JSONResponse {
|
||||
try {
|
||||
$this->prune->setPinned($this->targets->get((string)$targetId), $snapshotId, $pinned);
|
||||
} catch (\Throwable $e) {
|
||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||
}
|
||||
return new JSONResponse(['ok' => true]);
|
||||
}
|
||||
|
||||
public function browse(int $targetId, string $snapshotId, string $path = ''): JSONResponse {
|
||||
try {
|
||||
return new JSONResponse($this->restore->browse($this->targets->get((string)$targetId), $snapshotId, $path));
|
||||
@@ -301,7 +367,7 @@ final class AdminApiController extends Controller {
|
||||
if ($r->getSnapshotId() !== null) {
|
||||
try {
|
||||
$s = $this->snapshots->findOne($r->getTargetId(), $r->getSnapshotId());
|
||||
$snapshot = ['createdAt' => $s->getCreatedAt(), 'label' => $s->getLabel()];
|
||||
$snapshot = ['createdAt' => $s->getCreatedAt(), 'label' => $s->getLabel(), 'note' => $s->getNote(), 'pinned' => (bool)$s->getPinned()];
|
||||
} catch (\Throwable) {
|
||||
// Snapshot already pruned or trashed: the run keeps its id, just not the date.
|
||||
}
|
||||
@@ -309,6 +375,7 @@ final class AdminApiController extends Controller {
|
||||
return ['id' => $r->getId(), 'targetId' => $r->getTargetId(), 'kind' => $r->getKind(), 'status' => $r->getStatus(),
|
||||
'phase' => $r->getPhase(), 'snapshotId' => $r->getSnapshotId(), 'snapshot' => $snapshot, 'startedAt' => $r->getStartedAt(),
|
||||
'updatedAt' => $r->getUpdatedAt(), 'finishedAt' => $r->getFinishedAt(), 'error' => $r->getError(),
|
||||
'stats' => json_decode($r->getStats() ?? 'null', true), 'progress' => \OCA\NgBackup\Job\BackupRun::progressOf($r)];
|
||||
'stats' => json_decode($r->getStats() ?? 'null', true), 'progress' => \OCA\NgBackup\Job\BackupRun::progressOf($r),
|
||||
'waitingFor' => $r->getKind() === BackupService::KIND_FULL ? \OCA\NgBackup\Service\CronHealth::waitingFor($r) : null];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,14 +87,20 @@ final class KeyRing {
|
||||
foreach ($field['slots'] as $slot) {
|
||||
try {
|
||||
return self::unwrap($slot['key'], $passphrase);
|
||||
} catch (CryptoException) {
|
||||
} catch (CryptoException|\SodiumException|\TypeError|\ValueError) {
|
||||
// a wrong passphrase for this slot, or a damaged slot: the next one may still open
|
||||
}
|
||||
}
|
||||
throw new CryptoException('Wrong passphrase or damaged key');
|
||||
}
|
||||
|
||||
/** @param array{v:int, salt:string, ops:int, mem:int, nonce:string, key:string} $wrapped */
|
||||
/** @param array<string, mixed> $wrapped from a kit file or a location's config: checked here, not trusted */
|
||||
public static function unwrap(array $wrapped, #[\SensitiveParameter] string $passphrase): self {
|
||||
if (!isset($wrapped['salt'], $wrapped['nonce'], $wrapped['key'], $wrapped['ops'], $wrapped['mem'])
|
||||
|| !is_string($wrapped['salt']) || !is_string($wrapped['nonce']) || !is_string($wrapped['key'])
|
||||
|| !is_int($wrapped['ops']) || !is_int($wrapped['mem'])) {
|
||||
throw new CryptoException('Damaged key file: a part of the key is missing');
|
||||
}
|
||||
$b64 = static fn (string $s): string => sodium_base642bin($s, SODIUM_BASE64_VARIANT_ORIGINAL);
|
||||
// ops/mem come from unauthenticated JSON (kit file, repository config): bound them, so a
|
||||
// damaged or crafted file cannot make the KDF use unbounded memory or time.
|
||||
@@ -102,10 +108,22 @@ final class KeyRing {
|
||||
|| $wrapped['mem'] < SODIUM_CRYPTO_PWHASH_MEMLIMIT_INTERACTIVE || $wrapped['mem'] > SODIUM_CRYPTO_PWHASH_MEMLIMIT_SENSITIVE) {
|
||||
throw new CryptoException('Unsupported key derivation parameters (damaged or foreign key file)');
|
||||
}
|
||||
$kek = sodium_crypto_pwhash(SODIUM_CRYPTO_SECRETBOX_KEYBYTES, $passphrase, $b64($wrapped['salt']),
|
||||
$wrapped['ops'], $wrapped['mem'], SODIUM_CRYPTO_PWHASH_ALG_ARGON2ID13);
|
||||
$master = sodium_crypto_secretbox_open($b64($wrapped['key']), $b64($wrapped['nonce']), $kek);
|
||||
sodium_memzero($kek);
|
||||
$kek = null;
|
||||
try {
|
||||
$salt = $b64($wrapped['salt']);
|
||||
$sealed = $b64($wrapped['key']);
|
||||
$nonce = $b64($wrapped['nonce']);
|
||||
$kek = sodium_crypto_pwhash(SODIUM_CRYPTO_SECRETBOX_KEYBYTES, $passphrase, $salt,
|
||||
$wrapped['ops'], $wrapped['mem'], SODIUM_CRYPTO_PWHASH_ALG_ARGON2ID13);
|
||||
$master = sodium_crypto_secretbox_open($sealed, $nonce, $kek);
|
||||
} catch (\SodiumException|\ValueError $e) {
|
||||
// invalid base64 or a salt/nonce of the wrong length: damaged, so the caller shows its recovery options
|
||||
throw new CryptoException('Damaged key file: ' . $e->getMessage(), 0, $e);
|
||||
} finally {
|
||||
if ($kek !== null) {
|
||||
sodium_memzero($kek);
|
||||
}
|
||||
}
|
||||
if ($master === false) {
|
||||
throw new CryptoException('Wrong passphrase or damaged key');
|
||||
}
|
||||
|
||||
@@ -23,8 +23,12 @@ use OCA\NgBackup\Repository\Repository;
|
||||
* sets, so a single big SELECT would not stream).
|
||||
* - Each table becomes its own blob stream of JSON lines: a header line, then one JSON array per
|
||||
* row. A table that did not change produces the same blobs and costs nothing in the next run.
|
||||
* - The schema itself is not dumped: it comes from Nextcloud's migrations on the instance that is
|
||||
* restored into (same Nextcloud and app versions). The dump records those versions.
|
||||
* - The structure of each table (columns, types, primary key, indexes) is recorded in the manifest,
|
||||
* so a restore can create a table that does not exist on the new instance, for example one left
|
||||
* behind by an app that was removed long ago. Tables that exist keep the structure the new
|
||||
* instance's migrations gave them.
|
||||
* - Foreign keys are not checked while rows are loaded (tables are restored in name order, not in
|
||||
* dependency order), and are checked again before the restore commits.
|
||||
*
|
||||
* Phase 0 uses the Doctrine connection behind IDBConnection (private API) for schema
|
||||
* introspection; to be replaced by information_schema queries or a public API.
|
||||
@@ -108,7 +112,7 @@ final class DbDumper {
|
||||
break;
|
||||
}
|
||||
$state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'],
|
||||
'bytes' => $state['bytes'], 'sha256' => hash_final($hash)];
|
||||
'bytes' => $state['bytes'], 'sha256' => hash_final($hash), 'schema' => self::structure($schema)];
|
||||
$state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]);
|
||||
if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) {
|
||||
break;
|
||||
@@ -137,18 +141,48 @@ final class DbDumper {
|
||||
* Replace the contents of every table in the dump. Tables that exist but are not in the dump
|
||||
* are left alone (e.g. a newer app); the caller decides whether that is acceptable.
|
||||
*
|
||||
* @param array{tables: array<string, array{blobs: list<string>, rows: int}>} $manifest
|
||||
* @param array{tables: array<string, array{blobs: list<string>, rows: int, sha256: string, schema?: array}>, sequences?: array} $manifest
|
||||
*/
|
||||
public function restore(Repository $repo, array $manifest, ?array &$report = null, ?callable $heartbeat = null): int {
|
||||
$sm = $this->db->createSchemaManager();
|
||||
$existing = array_flip($sm->listTableNames());
|
||||
$total = 0;
|
||||
$report = ['skipped' => [], 'restored' => []];
|
||||
$this->db->beginTransaction();
|
||||
public function restore(Repository $repo, array $manifest, ?array &$report = null, ?callable $heartbeat = null, bool $skipMissing = false): int {
|
||||
$report = ['skipped' => [], 'restored' => [], 'created' => [], 'missing' => []];
|
||||
$source = self::dumpPrefix($manifest);
|
||||
if ($source !== null && $source !== $this->prefix) {
|
||||
// Every table would look missing and be created under the old prefix, unused.
|
||||
throw new \RuntimeException("The backup's tables use the prefix '$source', this installation '{$this->prefix}'. Install Nextcloud again with 'dbtableprefix' => '$source' in config/config.php before occ maintenance:install, and restore again.");
|
||||
}
|
||||
$missing = $this->missingTables($manifest);
|
||||
$unknown = array_values(array_filter($missing, fn (string $t) => !isset($manifest['tables'][$t]['schema'])));
|
||||
if ($unknown !== [] && !$skipMissing) {
|
||||
throw new \RuntimeException(self::describeMissing($unknown, $this->prefix));
|
||||
}
|
||||
// Tables the backup knows the structure of are created first (DDL is not transactional
|
||||
// everywhere); tables it does not know are left out, as asked.
|
||||
try {
|
||||
foreach ($missing as $table) {
|
||||
if (isset($manifest['tables'][$table]['schema'])) {
|
||||
$report['created'][] = $table; // first: creating it can stop halfway (its indexes)
|
||||
$this->db->createSchemaManager()->createTable(self::tableFromStructure($table, $manifest['tables'][$table]['schema'], $this->tableOptions()));
|
||||
} else {
|
||||
$report['missing'][] = $table;
|
||||
}
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
$this->dropCreatedTables($report);
|
||||
throw $e;
|
||||
}
|
||||
$total = 0;
|
||||
$foreignKeys = [];
|
||||
try {
|
||||
$this->db->beginTransaction();
|
||||
} catch (\Throwable $e) {
|
||||
$this->dropCreatedTables($report);
|
||||
throw $e;
|
||||
}
|
||||
try {
|
||||
$foreignKeys = $this->suspendForeignKeys(); // first: on SQLite it also takes the write lock
|
||||
foreach ($manifest['tables'] as $table => $info) {
|
||||
if (!isset($existing[$table])) {
|
||||
throw new \RuntimeException("Table $table from the backup does not exist here (different app versions?)");
|
||||
if (in_array($table, $report['missing'], true)) {
|
||||
continue;
|
||||
}
|
||||
// Unchanged since the backup: leave it alone (much faster than delete + insert).
|
||||
$live = $this->checksum($table);
|
||||
@@ -192,15 +226,253 @@ final class DbDumper {
|
||||
}
|
||||
$report['restored'][] = $table;
|
||||
}
|
||||
$this->restoreSequences($manifest['sequences'] ?? []);
|
||||
// unchanged tables too: a restore that stopped earlier may have left their counters at 1
|
||||
$this->restoreSequences($manifest['sequences'] ?? [], $report['missing'], array_merge($report['restored'], $report['skipped']));
|
||||
$this->resumeForeignKeys($foreignKeys);
|
||||
$this->db->commit();
|
||||
} catch (\Throwable $e) {
|
||||
$this->db->rollBack();
|
||||
// PostgreSQL rolls the dropped foreign keys back with the transaction; MySQL's session
|
||||
// switch must be turned on again by hand.
|
||||
if (in_array('SET FOREIGN_KEY_CHECKS = 1', $foreignKeys, true)) {
|
||||
$this->db->executeStatement('SET FOREIGN_KEY_CHECKS = 1');
|
||||
}
|
||||
$this->dropCreatedTables($report);
|
||||
throw $e;
|
||||
}
|
||||
// Committed. Foreign keys of the tables created above, now their rows are in. Not inside the
|
||||
// transaction: MySQL commits on DDL. A failure is reported, not fatal: the data is restored
|
||||
// either way, and the created tables stay.
|
||||
foreach ($report['created'] as $table) {
|
||||
$structure = $manifest['tables'][$table]['schema'] ?? [];
|
||||
foreach ($structure['foreignKeys'] ?? [] as $fk) {
|
||||
try {
|
||||
$this->db->createSchemaManager()->createForeignKey(
|
||||
new \Doctrine\DBAL\Schema\ForeignKeyConstraint($fk['columns'], $fk['table'], $fk['foreign'], $fk['name'], $fk['options'] ?? []),
|
||||
$table);
|
||||
} catch (\Throwable $e) {
|
||||
$report['foreignKeyErrors'][] = $table . '.' . $fk['name'] . ': ' . $e->getMessage();
|
||||
}
|
||||
}
|
||||
}
|
||||
return $total;
|
||||
}
|
||||
|
||||
/**
|
||||
* The restore did not go through: a rollback does not undo CREATE TABLE everywhere. Left behind,
|
||||
* the empty tables of an app would stop that app's installation later ("table already exists").
|
||||
*
|
||||
* @param array{created: list<string>, ...} $report
|
||||
*/
|
||||
private function dropCreatedTables(array &$report): void {
|
||||
foreach ($report['created'] as $table) {
|
||||
try {
|
||||
$this->db->createSchemaManager()->dropTable($table);
|
||||
} catch (\Throwable) {
|
||||
// the error that brought us here is the one to report; a run again uses the table
|
||||
}
|
||||
}
|
||||
$report['created'] = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* The table prefix of the instance the dump was made on (from its appconfig table), or null
|
||||
* when the dump has none.
|
||||
*
|
||||
*/
|
||||
public static function dumpPrefix(array $manifest): ?string {
|
||||
/** @var list<string> $tables table names */
|
||||
$tables = array_keys($manifest['tables'] ?? []);
|
||||
foreach ($tables as $name) {
|
||||
if (str_ends_with($name, 'appconfig') && !str_ends_with($name, '_ex_appconfig')) {
|
||||
return substr($name, 0, -strlen('appconfig'));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tables in the dump that do not exist on this instance.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
public function missingTables(array $manifest): array {
|
||||
$existing = array_flip($this->db->createSchemaManager()->listTableNames());
|
||||
return array_values(array_filter(array_keys($manifest['tables']), fn (string $t) => !isset($existing[$t])));
|
||||
}
|
||||
|
||||
/** One message for all missing tables of an older backup, grouped by the app they probably belong to. */
|
||||
public static function describeMissing(array $tables, string $prefix): string {
|
||||
$groups = [];
|
||||
foreach ($tables as $t) {
|
||||
$name = substr($t, strlen($prefix));
|
||||
$groups[explode('_', $name, 2)[0]][] = $t;
|
||||
}
|
||||
$lines = [];
|
||||
foreach ($groups as $app => $list) {
|
||||
$lines[] = '- ' . $app . ': ' . implode(', ', $list);
|
||||
}
|
||||
return "These tables from the backup do not exist here, and this backup does not record their structure (made before 1.0.0-rc.2):\n"
|
||||
. implode("\n", $lines)
|
||||
. "\nInstall the apps they belong to (often apps that were removed or switched off long ago) and run the restore again, or leave these tables out with --skip-missing-tables.";
|
||||
}
|
||||
|
||||
/**
|
||||
* Portable description of a table: enough to create it again, also on another database type.
|
||||
*
|
||||
* @return array{columns: list<array<string, mixed>>, pk: list<string>, indexes: list<array{name: string, columns: list<string>, unique: bool}>}
|
||||
*/
|
||||
private static function structure(\Doctrine\DBAL\Schema\Table $table): array {
|
||||
$columns = [];
|
||||
foreach ($table->getColumns() as $c) {
|
||||
$columns[] = ['name' => $c->getName(), 'type' => \Doctrine\DBAL\Types\Type::lookupName($c->getType()),
|
||||
'length' => $c->getLength(), 'precision' => $c->getPrecision(), 'scale' => $c->getScale(),
|
||||
'unsigned' => $c->getUnsigned(), 'fixed' => $c->getFixed(), 'notnull' => $c->getNotnull(),
|
||||
'default' => $c->getDefault(), 'autoincrement' => $c->getAutoincrement()];
|
||||
}
|
||||
$indexes = [];
|
||||
foreach ($table->getIndexes() as $i) {
|
||||
if (!$i->isPrimary()) {
|
||||
// options holds the prefix lengths MySQL needs for indexes on long text columns
|
||||
$indexes[] = ['name' => $i->getName(), 'columns' => array_values($i->getColumns()), 'unique' => $i->isUnique(),
|
||||
'flags' => array_values($i->getFlags()), 'options' => $i->getOptions()];
|
||||
}
|
||||
}
|
||||
$foreignKeys = [];
|
||||
foreach ($table->getForeignKeys() as $fk) {
|
||||
$foreignKeys[] = ['name' => $fk->getName(), 'columns' => array_values($fk->getLocalColumns()),
|
||||
'table' => $fk->getForeignTableName(), 'foreign' => array_values($fk->getForeignColumns()), 'options' => $fk->getOptions()];
|
||||
}
|
||||
return ['columns' => $columns, 'pk' => array_values($table->getPrimaryKey()?->getColumns() ?? []), 'indexes' => $indexes,
|
||||
'foreignKeys' => $foreignKeys];
|
||||
}
|
||||
|
||||
/**
|
||||
* Table options for a table the restore creates: those of this installation's own tables.
|
||||
* On MySQL/MariaDB that is the character set and collation, as Nextcloud's tables have them
|
||||
* (utf8mb4, so emoji and other 4-byte characters fit); DBAL alone would use 3-byte utf8.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
* @psalm-suppress UndefinedClass the DBAL classes come with Nextcloud, not with this app
|
||||
*/
|
||||
private function tableOptions(): array {
|
||||
$options = $this->db->createSchemaManager()->createSchemaConfig()->getDefaultTableOptions();
|
||||
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\AbstractMySQLPlatform) {
|
||||
$collation = $this->db->fetchOne('SELECT TABLE_COLLATION FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?',
|
||||
[$this->prefix . 'appconfig']);
|
||||
if (is_string($collation) && $collation !== '') {
|
||||
$options['collation'] = $collation;
|
||||
$options['charset'] = explode('_', $collation, 2)[0];
|
||||
unset($options['collate']);
|
||||
}
|
||||
}
|
||||
return $options;
|
||||
}
|
||||
|
||||
/** @param array<string, mixed> $options table options (see tableOptions()) */
|
||||
private static function tableFromStructure(string $name, array $structure, array $options = []): \Doctrine\DBAL\Schema\Table {
|
||||
$table = new \Doctrine\DBAL\Schema\Table($name, [], [], [], [], $options);
|
||||
foreach ($structure['columns'] as $c) {
|
||||
$table->addColumn($c['name'], $c['type'], array_filter([
|
||||
'length' => $c['length'], 'precision' => $c['precision'], 'scale' => $c['scale'],
|
||||
'unsigned' => $c['unsigned'], 'fixed' => $c['fixed'], 'notnull' => $c['notnull'],
|
||||
'default' => $c['default'], 'autoincrement' => $c['autoincrement'],
|
||||
], fn ($v) => $v !== null));
|
||||
}
|
||||
if ($structure['pk'] !== []) {
|
||||
$table->setPrimaryKey($structure['pk']);
|
||||
}
|
||||
foreach ($structure['indexes'] as $i) {
|
||||
$i['unique']
|
||||
? $table->addUniqueIndex($i['columns'], $i['name'], $i['options'] ?? [])
|
||||
: $table->addIndex($i['columns'], $i['name'], $i['flags'] ?? [], $i['options'] ?? []);
|
||||
}
|
||||
return $table;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop checking foreign keys while rows are loaded in name order. PostgreSQL has no session
|
||||
* switch for this without superuser rights, so its foreign keys of Nextcloud tables are dropped
|
||||
* here and added back (and so checked) by resumeForeignKeys(), inside the same transaction.
|
||||
*
|
||||
* @return list<string> statements that restore the foreign keys
|
||||
*/
|
||||
private function suspendForeignKeys(): array {
|
||||
$platform = $this->db->getDatabasePlatform();
|
||||
if ($platform instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
|
||||
$rows = $this->db->fetchAllAssociative(
|
||||
"SELECT conrelid::regclass::text AS tbl, conname, pg_get_constraintdef(oid) AS def
|
||||
FROM pg_constraint WHERE contype = 'f' AND connamespace = current_schema()::regnamespace");
|
||||
$restore = [];
|
||||
foreach ($rows as $r) {
|
||||
if (!str_starts_with(trim($r['tbl'], '"'), $this->prefix)) {
|
||||
continue;
|
||||
}
|
||||
$this->db->executeStatement(sprintf('ALTER TABLE %s DROP CONSTRAINT %s', $r['tbl'], $this->db->quoteIdentifier($r['conname'])));
|
||||
$restore[] = sprintf('ALTER TABLE %s ADD CONSTRAINT %s %s', $r['tbl'], $this->db->quoteIdentifier($r['conname']), $r['def']);
|
||||
}
|
||||
return $restore;
|
||||
}
|
||||
if ($platform instanceof \Doctrine\DBAL\Platforms\AbstractMySQLPlatform) {
|
||||
// MySQL does not check rows again when FOREIGN_KEY_CHECKS goes back to 1, and its DDL
|
||||
// would commit the transaction halfway. So every foreign key is checked explicitly,
|
||||
// with a query that finds rows pointing at nothing (resumeForeignKeys()).
|
||||
$rows = $this->db->fetchAllAssociative(
|
||||
'SELECT TABLE_NAME AS tbl, CONSTRAINT_NAME AS name, COLUMN_NAME AS col, REFERENCED_TABLE_NAME AS rtbl, REFERENCED_COLUMN_NAME AS rcol
|
||||
FROM information_schema.KEY_COLUMN_USAGE
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND REFERENCED_TABLE_NAME IS NOT NULL
|
||||
ORDER BY TABLE_NAME, CONSTRAINT_NAME, ORDINAL_POSITION');
|
||||
$keys = [];
|
||||
foreach ($rows as $r) {
|
||||
if (str_starts_with((string)$r['tbl'], $this->prefix)) {
|
||||
$keys[$r['tbl'] . "\0" . $r['name']][] = $r;
|
||||
}
|
||||
}
|
||||
$q = fn (string $i): string => $this->db->quoteIdentifier($i);
|
||||
$checks = ['SET FOREIGN_KEY_CHECKS = 1'];
|
||||
foreach ($keys as $parts) {
|
||||
$join = implode(' AND ', array_map(fn (array $r): string => 'c.' . $q($r['col']) . ' = p.' . $q($r['rcol']), $parts));
|
||||
$set = implode(' AND ', array_map(fn (array $r): string => 'c.' . $q($r['col']) . ' IS NOT NULL', $parts));
|
||||
$checks[] = sprintf('SELECT COUNT(*) FROM %s c LEFT JOIN %s p ON %s WHERE %s AND p.%s IS NULL /* %s */',
|
||||
$q($parts[0]['tbl']), $q($parts[0]['rtbl']), $join, $set, $q($parts[0]['rcol']), $parts[0]['name']);
|
||||
}
|
||||
$this->db->executeStatement('SET FOREIGN_KEY_CHECKS = 0');
|
||||
return $checks;
|
||||
}
|
||||
if ($platform instanceof \Doctrine\DBAL\Platforms\SqlitePlatform) {
|
||||
// SQLite (WAL) starts a transaction as a reader. When another process (a cron job, the
|
||||
// installer still finishing) writes between our first read and our first write, that
|
||||
// write fails at once with "database is locked": no waiting helps a reader whose view
|
||||
// is out of date. Take the write lock first, so we wait for the other writer instead
|
||||
// and it waits for us after that.
|
||||
$this->db->executeStatement('DELETE FROM ' . $this->db->quoteIdentifier($this->prefix . 'appconfig') . ' WHERE 0 = 1');
|
||||
$this->db->executeStatement('PRAGMA defer_foreign_keys = ON'); // checked at commit
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the foreign keys again; a row that breaks one fails the restore here, before it commits.
|
||||
* Statements add them back (PostgreSQL) or switch the checks on (MySQL); queries count the rows
|
||||
* that point at nothing (MySQL) and must find none.
|
||||
*/
|
||||
private function resumeForeignKeys(array $statements): void {
|
||||
foreach ($statements as $sql) {
|
||||
try {
|
||||
if (str_starts_with($sql, 'SELECT ')) {
|
||||
$orphans = (int)$this->db->fetchOne($sql);
|
||||
if ($orphans > 0) {
|
||||
throw new \RuntimeException("$orphans rows point at nothing");
|
||||
}
|
||||
} else {
|
||||
$this->db->executeStatement($sql);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
throw new \RuntimeException('The restored data breaks a foreign key (' . $sql . '): ' . $e->getMessage(), 0, $e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Same checksum as dump(): sha256 over the row lines in primary-key order. */
|
||||
public function checksum(string $table): array {
|
||||
$hash = hash_init('sha256');
|
||||
@@ -359,11 +631,21 @@ final class DbDumper {
|
||||
return $name;
|
||||
}
|
||||
|
||||
private function restoreSequences(array $sequences): void {
|
||||
/**
|
||||
* @param list<string> $leftOut tables not restored (--skip-missing-tables): they do not exist here
|
||||
* @param list<string> $restored tables restored or unchanged: their other counters go after their data
|
||||
*/
|
||||
private function restoreSequences(array $sequences, array $leftOut = [], array $restored = []): void {
|
||||
if (!$this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
|
||||
return;
|
||||
}
|
||||
$leftOut = array_flip($leftOut);
|
||||
$set = [];
|
||||
foreach ($sequences as $info) {
|
||||
if (isset($leftOut[$info['table']])) {
|
||||
continue;
|
||||
}
|
||||
$set[$info['table'] . '.' . $info['column']] = true;
|
||||
$seq = $this->db->fetchOne('SELECT pg_get_serial_sequence(quote_ident(?), ?)', [$info['table'], $info['column']]);
|
||||
if ($seq === false || $seq === null) {
|
||||
continue;
|
||||
@@ -378,5 +660,26 @@ final class DbDumper {
|
||||
[\Doctrine\DBAL\ParameterType::STRING, \Doctrine\DBAL\ParameterType::INTEGER, \Doctrine\DBAL\ParameterType::BOOLEAN]);
|
||||
}
|
||||
}
|
||||
// A backup from MySQL/MariaDB or SQLite records no counters (those derive them from the
|
||||
// data): every other counter of a restored table goes after its data, or new rows (a new
|
||||
// file, an activity) would collide with restored ones.
|
||||
$restored = array_flip($restored);
|
||||
$rows = $this->db->fetchAllAssociative(
|
||||
"SELECT c.table_name, c.column_name, pg_get_serial_sequence(quote_ident(c.table_name), c.column_name) AS seq
|
||||
FROM information_schema.columns c
|
||||
WHERE c.table_schema = current_schema() AND pg_get_serial_sequence(quote_ident(c.table_name), c.column_name) IS NOT NULL");
|
||||
foreach ($rows as $r) {
|
||||
if (!isset($restored[$r['table_name']]) || isset($set[$r['table_name'] . '.' . $r['column_name']])) {
|
||||
continue;
|
||||
}
|
||||
$max = (int)$this->db->fetchOne(sprintf('SELECT COALESCE(MAX(%s), 0) FROM %s',
|
||||
$this->db->quoteIdentifier($r['column_name']), $this->db->quoteIdentifier($r['table_name'])));
|
||||
// never backwards: a counter already past the data stays where it is
|
||||
$state = $this->db->fetchAssociative('SELECT last_value, is_called FROM ' . self::sequenceIdentifier($r['seq']));
|
||||
$next = $state === false ? 1 : (int)$state['last_value'] + ($state['is_called'] ? 1 : 0);
|
||||
if ($next <= $max) {
|
||||
$this->db->executeQuery('SELECT setval(CAST(? AS regclass), ?, false)', [$r['seq'], $max + 1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,6 +64,13 @@ final class DbIndexCache implements IIndexCache {
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop the cached index of a repository that no location uses any more. */
|
||||
public function forget(string $repositoryId): void {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->delete('ngb_index')->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId)));
|
||||
$qb->executeStatement();
|
||||
}
|
||||
|
||||
/** All or nothing: when it fails, the cache stays as it was. */
|
||||
public function replaceAll(string $repositoryId, array $packs): void {
|
||||
$this->transaction(function () use ($repositoryId, $packs): void {
|
||||
|
||||
@@ -24,6 +24,10 @@ use OCP\AppFramework\Db\Entity;
|
||||
* @method void setFiles(?int $v)
|
||||
* @method int|null getBytes()
|
||||
* @method void setBytes(?int $v)
|
||||
* @method string|null getNote()
|
||||
* @method void setNote(?string $v)
|
||||
* @method bool|null getPinned()
|
||||
* @method void setPinned(?bool $v)
|
||||
*/
|
||||
final class Snapshot extends Entity {
|
||||
protected int $targetId = 0;
|
||||
@@ -33,8 +37,11 @@ final class Snapshot extends Entity {
|
||||
protected int $createdAt = 0;
|
||||
protected ?int $files = null;
|
||||
protected ?int $bytes = null;
|
||||
protected ?string $note = null;
|
||||
protected ?bool $pinned = null;
|
||||
|
||||
public function __construct() {
|
||||
$this->addType('pinned', 'boolean');
|
||||
$this->addType('targetId', 'integer');
|
||||
$this->addType('createdAt', 'integer');
|
||||
$this->addType('files', 'integer');
|
||||
|
||||
@@ -28,6 +28,12 @@ use OCP\AppFramework\Db\Entity;
|
||||
* @method void setMaxBytes(?int $v)
|
||||
* @method int getCreatedAt()
|
||||
* @method void setCreatedAt(int $v)
|
||||
* @method string|null getSchedule()
|
||||
* @method void setSchedule(?string $v)
|
||||
* @method string|null getScheduleZone()
|
||||
* @method void setScheduleZone(?string $v)
|
||||
* @method int|null getScheduleSince()
|
||||
* @method void setScheduleSince(?int $v)
|
||||
*/
|
||||
final class Target extends Entity {
|
||||
protected string $name = '';
|
||||
@@ -39,8 +45,12 @@ final class Target extends Entity {
|
||||
protected int $createdAt = 0;
|
||||
protected ?bool $appendOnly = null;
|
||||
protected ?int $maxBytes = null;
|
||||
protected ?string $schedule = null;
|
||||
protected ?string $scheduleZone = null;
|
||||
protected ?int $scheduleSince = null;
|
||||
|
||||
public function __construct() {
|
||||
$this->addType('scheduleSince', 'integer');
|
||||
$this->addType('createdAt', 'integer');
|
||||
$this->addType('appendOnly', 'boolean');
|
||||
$this->addType('maxBytes', 'integer');
|
||||
|
||||
@@ -15,4 +15,14 @@ final class Format {
|
||||
$time = gmdate('H:i:s', $seconds % 86400);
|
||||
return $days > 0 ? $days . 'd ' . $time : $time;
|
||||
}
|
||||
|
||||
/**
|
||||
* "2026-10-29 04:30" in $zone. Nextcloud runs PHP in UTC; occ shows times in the zone of the
|
||||
* schedule, the administrator's, as the web interface does.
|
||||
*
|
||||
* @param non-empty-string $zone
|
||||
*/
|
||||
public static function time(int $timestamp, string $zone): string {
|
||||
return (new \DateTimeImmutable('@' . $timestamp))->setTimezone(new \DateTimeZone($zone))->format('Y-m-d H:i');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,12 +52,51 @@ final class BackupRun {
|
||||
$stats = $state['stats'];
|
||||
$walker = new TreeWalker($state['roots'], $state['exclude'], function (string $logical) use (&$state, &$stats): void {
|
||||
self::recordUnreadable($state, $stats, $logical);
|
||||
});
|
||||
}, true);
|
||||
$parent = new ParentCursor($repo, $state['parent'], $state['parentPos']);
|
||||
$lines = [];
|
||||
$emptyDirs = [];
|
||||
$folders = [];
|
||||
$finishedWalk = true;
|
||||
|
||||
foreach ($walker->walk($state['after']) as $logical => $path) {
|
||||
$safe = mb_check_encoding($logical, 'UTF-8') && Repository::isSafePath($logical);
|
||||
if ($walker->atFolderWithContent()) {
|
||||
// Only its time: what is in it is reported by itself, and a bad name in the path of
|
||||
// each of those is what gets reported. The cursor stays for such a name, see below.
|
||||
$mtime = $safe ? @filemtime($path) : false;
|
||||
if ($safe && $mtime !== false) {
|
||||
$folders[] = json_encode([$logical, $mtime], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
||||
$state['after'] = $logical;
|
||||
if (microtime(true) >= $deadline) {
|
||||
$finishedWalk = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!$safe) {
|
||||
// A name a snapshot cannot hold (a backslash, a control character, not UTF-8). One
|
||||
// such path must not make a backup fail, or the whole snapshot unreadable later.
|
||||
// The cursor stays where it is: the state is JSON, which cannot hold such a name. A
|
||||
// next step meets it again, and it is reported once.
|
||||
self::recordBadName($state, $stats, $logical);
|
||||
continue;
|
||||
}
|
||||
if (is_dir($path)) {
|
||||
$mtime = @filemtime($path);
|
||||
if ($mtime === false) {
|
||||
continue; // removed while walking
|
||||
}
|
||||
$emptyDirs[] = json_encode([$logical, $mtime], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
||||
$stats['emptyDirs'] = ($stats['emptyDirs'] ?? 0) + 1;
|
||||
$state['after'] = $logical;
|
||||
if (microtime(true) >= $deadline) {
|
||||
$finishedWalk = false;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
$version = Repository::fileVersion($path);
|
||||
if ($version === 'missing') {
|
||||
continue; // deleted while walking
|
||||
@@ -81,7 +120,7 @@ final class BackupRun {
|
||||
if ($cur === null || $cur['path'] !== $logical) {
|
||||
$cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version];
|
||||
}
|
||||
$fh = @fopen($path, 'rb');
|
||||
$fh = Repository::openForReading($path);
|
||||
if ($fh === false) {
|
||||
self::recordUnreadable($state, $stats, $logical);
|
||||
$state['cur'] = null;
|
||||
@@ -136,6 +175,16 @@ final class BackupRun {
|
||||
$writer->write(implode("\n", $lines) . "\n");
|
||||
$state['segments'][] = $writer->finish();
|
||||
}
|
||||
if ($emptyDirs !== []) {
|
||||
$writer = $repo->blobWriter($stats);
|
||||
$writer->write(implode("\n", $emptyDirs) . "\n");
|
||||
$state['dirSegments'][] = $writer->finish();
|
||||
}
|
||||
if ($folders !== []) {
|
||||
$writer = $repo->blobWriter($stats);
|
||||
$writer->write(implode("\n", $folders) . "\n");
|
||||
$state['folderSegments'][] = $writer->finish();
|
||||
}
|
||||
$repo->flushPacks();
|
||||
$state['parentPos'] = $parent->position();
|
||||
$state['stats'] = $stats;
|
||||
@@ -150,10 +199,22 @@ final class BackupRun {
|
||||
if ($state['phase'] === 'finish') {
|
||||
// Every segment ends with a newline, so they read as one stream: fetched together,
|
||||
// several at a time where the location allows it.
|
||||
// Empty folders go in the meta record, not between the files: a version that does not
|
||||
// know them reads the snapshot as before, and only restores no empty folders.
|
||||
$emptyDirs = [];
|
||||
foreach (isset($state['dirSegments']) ? $repo->readLines(array_values(array_merge(...$state['dirSegments']))) : [] as $line) {
|
||||
if ($line !== '') {
|
||||
$emptyDirs[] = json_decode($line, true, 8, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
}
|
||||
// The times of the other folders too, packed: every look at a snapshot's meta record reads
|
||||
// this, and a string of a few hundred KiB costs that far less than tens of thousands of arrays.
|
||||
$folders = isset($state['folderSegments']) ? implode("\n", iterator_to_array($repo->readLines(array_values(array_merge(...$state['folderSegments']))), false)) : '';
|
||||
$repo->putSnapshot($state['snapshot'],
|
||||
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
|
||||
'roots' => array_keys($state['roots']), 'stats' => $state['stats'],
|
||||
'warnings' => self::warnings($state)] + $state['meta'],
|
||||
'warnings' => self::warnings($state)] + ($emptyDirs === [] ? [] : ['emptyDirs' => $emptyDirs])
|
||||
+ ($folders === '' ? [] : ['folderTimes' => base64_encode((string)gzdeflate($folders))]) + $state['meta'],
|
||||
$repo->readLines(array_values(array_merge(...$state['segments']))));
|
||||
$state['phase'] = 'commit';
|
||||
}
|
||||
@@ -176,12 +237,12 @@ final class BackupRun {
|
||||
}
|
||||
|
||||
/**
|
||||
* progress() of a run, while it is in its files phase.
|
||||
* progress() of a run in its files phase; in its database phase, the tables dumped so far.
|
||||
*
|
||||
* @return array{fraction: float, approximate: bool}|null
|
||||
* @return array{fraction: float, approximate: bool, tables?: int, totalTables?: int}|null
|
||||
*/
|
||||
public static function progressOf(\OCA\NgBackup\Db\Run $run): ?array {
|
||||
if ($run->getStatus() !== \OCA\NgBackup\Db\Run::RUNNING || $run->getPhase() !== 'files') {
|
||||
if ($run->getStatus() !== \OCA\NgBackup\Db\Run::RUNNING || !in_array($run->getPhase(), ['db', 'files'], true)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
@@ -189,6 +250,14 @@ final class BackupRun {
|
||||
} catch (\JsonException) {
|
||||
return null;
|
||||
}
|
||||
if ($run->getPhase() === 'db') {
|
||||
// Tables dumped so far, as of the last step (none known before the first step ends)
|
||||
$tables = count($state['dump']['tables'] ?? []);
|
||||
return $tables > 0
|
||||
? ['fraction' => min(1.0, ($state['dump']['i'] ?? 0) / $tables), 'approximate' => false,
|
||||
'tables' => (int)($state['dump']['i'] ?? 0), 'totalTables' => $tables]
|
||||
: null;
|
||||
}
|
||||
return is_array($state['files'] ?? null) ? self::progress($state['files']) : null;
|
||||
}
|
||||
|
||||
@@ -206,10 +275,26 @@ final class BackupRun {
|
||||
$stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1;
|
||||
}
|
||||
|
||||
/** @return array{unreadable: list<string>, changing: list<string>} examples for the report (first 50 each) */
|
||||
/**
|
||||
* A file or directory left out because of its name. Remembered like an unreadable one, in a form
|
||||
* that fits the JSON state and is safe to show: control characters and invalid bytes replaced.
|
||||
*/
|
||||
private static function recordBadName(array &$state, array &$stats, string $logical): void {
|
||||
$shown = (string)preg_replace('/[\x00-\x1f\x7f]/', '?', mb_scrub($logical, 'UTF-8'));
|
||||
if (isset($state['badNames'][$shown])) {
|
||||
return;
|
||||
}
|
||||
if (count($state['badNames'] ?? []) < 1000) {
|
||||
$state['badNames'][$shown] = true;
|
||||
}
|
||||
$stats['badNames'] = ($stats['badNames'] ?? 0) + 1;
|
||||
}
|
||||
|
||||
/** @return array{unreadable: list<string>, badNames: list<string>, changing: list<string>} examples for the report (first 50 each) */
|
||||
public static function warnings(array $state): array {
|
||||
return [
|
||||
'unreadable' => array_slice(array_map('strval', array_keys($state['unreadable'] ?? [])), 0, 50),
|
||||
'badNames' => array_slice(array_map('strval', array_keys($state['badNames'] ?? [])), 0, 50),
|
||||
'changing' => array_slice(array_map('strval', array_values($state['changing'] ?? [])), 0, 50),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -9,12 +9,16 @@ namespace OCA\NgBackup\Job;
|
||||
|
||||
/**
|
||||
* Walks one or more source roots in a stable depth-first order (entries of a directory sorted
|
||||
* by name, children right after their directory), yielding "<root>/<relative path>" for files.
|
||||
* by name, children right after their directory), yielding "<root>/<relative path>" for files
|
||||
* and for directories with nothing in them to walk (the caller tells them apart with is_dir()).
|
||||
* A root itself without a name is never yielded.
|
||||
*
|
||||
* The same order is used for comparing paths (compare()), so a walk can resume after a cursor
|
||||
* and be merge-joined with the previous snapshot without holding any list in memory.
|
||||
*/
|
||||
final class TreeWalker {
|
||||
private bool $atFolder = false;
|
||||
|
||||
/**
|
||||
* @param array<string, string> $roots name => absolute directory, walked in this order
|
||||
* @param list<string> $exclude path prefixes ("<root>/<relative>") that are skipped entirely;
|
||||
@@ -25,9 +29,16 @@ final class TreeWalker {
|
||||
private array $exclude = [],
|
||||
/** Called with the logical path of a directory that cannot be listed. */
|
||||
private ?\Closure $onUnreadable = null,
|
||||
/** Also yield directories with something in them, right before the first thing in them. */
|
||||
private bool $withFolders = false,
|
||||
) {
|
||||
}
|
||||
|
||||
/** Whether what was just yielded is a directory with something in it (see $withFolders). */
|
||||
public function atFolderWithContent(): bool {
|
||||
return $this->atFolder;
|
||||
}
|
||||
|
||||
/** @return \Generator<string, string> logical path => absolute path, strictly after $after */
|
||||
public function walk(?string $after = null): \Generator {
|
||||
foreach ($this->roots as $name => $dir) {
|
||||
@@ -66,27 +77,41 @@ final class TreeWalker {
|
||||
}
|
||||
$names = array_values(array_diff($names, ['.', '..']));
|
||||
sort($names, SORT_STRING);
|
||||
$empty = true; // no file or directory below it that is walked, whatever the cursor skips
|
||||
foreach ($names as $name) {
|
||||
$childLogical = $logical === '' ? $name : $logical . '/' . $name;
|
||||
$childAbs = $abs . '/' . $name;
|
||||
if (is_link($childAbs)) {
|
||||
continue; // never follow symlinks out of the data directory
|
||||
}
|
||||
if (is_dir($childAbs)) {
|
||||
$childIsDir = is_dir($childAbs);
|
||||
if (($childIsDir || is_file($childAbs)) && $empty && !$this->isExcluded($childLogical, $childIsDir)
|
||||
&& $this->withFolders && $logical !== '' && ($after === null || self::compare($logical, $after) > 0)) {
|
||||
$this->atFolder = true;
|
||||
yield $logical => $abs;
|
||||
$this->atFolder = false;
|
||||
}
|
||||
if ($childIsDir) {
|
||||
$empty = $empty && $this->isExcluded($childLogical, true);
|
||||
// Skip a directory entirely if everything in it sorts before the cursor.
|
||||
if ($after !== null && self::compare($childLogical, $after) < 0 && !self::isPrefix($childLogical, $after)) {
|
||||
continue;
|
||||
}
|
||||
yield from $this->walkDir($childAbs, $childLogical, $after);
|
||||
} elseif (is_file($childAbs)) {
|
||||
$excluded = $this->isExcluded($childLogical, false);
|
||||
$empty = $empty && $excluded;
|
||||
if ($after !== null && self::compare($childLogical, $after) <= 0) {
|
||||
continue;
|
||||
}
|
||||
if (!$this->isExcluded($childLogical, false)) {
|
||||
if (!$excluded) {
|
||||
yield $childLogical => $childAbs;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($empty && $logical !== '' && ($after === null || self::compare($logical, $after) > 0)) {
|
||||
yield $logical => $abs;
|
||||
}
|
||||
}
|
||||
|
||||
private function isExcluded(string $logical, bool $isDir): bool {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Migration;
|
||||
|
||||
use Closure;
|
||||
use OCP\IDBConnection;
|
||||
use OCP\Migration\IOutput;
|
||||
use OCP\Migration\SimpleMigrationStep;
|
||||
|
||||
/**
|
||||
* Removing a location left the cached index of its repository behind (tens of thousands of rows
|
||||
* for a large backup). Drop the cache of repositories no location uses any more.
|
||||
*/
|
||||
final class Version000006Date20261018000000 extends SimpleMigrationStep {
|
||||
public function __construct(
|
||||
private IDBConnection $db,
|
||||
) {
|
||||
}
|
||||
|
||||
public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
|
||||
$used = $this->db->getQueryBuilder();
|
||||
$used->select('repository_id')->from('ngb_targets')->where($used->expr()->isNotNull('repository_id'));
|
||||
$keep = array_column($used->executeQuery()->fetchAll(), 'repository_id');
|
||||
$delete = $this->db->getQueryBuilder();
|
||||
$delete->delete('ngb_index');
|
||||
if ($keep !== []) {
|
||||
$delete->where($delete->expr()->notIn('repository_id', $delete->createNamedParameter($keep, \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_STR_ARRAY)));
|
||||
}
|
||||
$removed = $delete->executeStatement();
|
||||
if ($removed > 0) {
|
||||
$output->info("Removed $removed cached index rows of locations that no longer exist");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Migration;
|
||||
|
||||
use Closure;
|
||||
use OCP\DB\ISchemaWrapper;
|
||||
use OCP\DB\Types;
|
||||
use OCP\Migration\IOutput;
|
||||
use OCP\Migration\SimpleMigrationStep;
|
||||
|
||||
/** Local copy of what the catalog on the location says about a snapshot: its note and pin. */
|
||||
final class Version000007Date20261107000000 extends SimpleMigrationStep {
|
||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||
/** @var ISchemaWrapper $schema */
|
||||
$schema = $schemaClosure();
|
||||
$t = $schema->getTable('ngb_snapshots');
|
||||
$changed = false;
|
||||
if (!$t->hasColumn('note')) {
|
||||
$t->addColumn('note', Types::STRING, ['notnull' => false, 'length' => 1000]);
|
||||
$changed = true;
|
||||
}
|
||||
if (!$t->hasColumn('pinned')) {
|
||||
$t->addColumn('pinned', Types::BOOLEAN, ['notnull' => false, 'default' => false]);
|
||||
$changed = true;
|
||||
}
|
||||
return $changed ? $schema : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Migration;
|
||||
|
||||
use Closure;
|
||||
use OCA\NgBackup\Service\LocationSchedule;
|
||||
use OCA\NgBackup\Service\Schedule;
|
||||
use OCP\DB\ISchemaWrapper;
|
||||
use OCP\DB\Types;
|
||||
use OCP\IDBConnection;
|
||||
use OCP\Migration\IOutput;
|
||||
use OCP\Migration\SimpleMigrationStep;
|
||||
|
||||
/** A schedule per location. Each existing location gets the daily time that held for all of them. */
|
||||
final class Version000008Date20261107010000 extends SimpleMigrationStep {
|
||||
public function __construct(
|
||||
private IDBConnection $db,
|
||||
private Schedule $schedule,
|
||||
) {
|
||||
}
|
||||
|
||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||
/** @var ISchemaWrapper $schema */
|
||||
$schema = $schemaClosure();
|
||||
$t = $schema->getTable('ngb_targets');
|
||||
if ($t->hasColumn('schedule')) {
|
||||
return null;
|
||||
}
|
||||
$t->addColumn('schedule', Types::STRING, ['notnull' => false, 'length' => 16]);
|
||||
$t->addColumn('schedule_zone', Types::STRING, ['notnull' => false, 'length' => 64]);
|
||||
$t->addColumn('schedule_since', Types::BIGINT, ['notnull' => false, 'unsigned' => true]);
|
||||
return $schema;
|
||||
}
|
||||
|
||||
public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
|
||||
['time' => $time, 'timezone' => $zone] = $this->schedule->get();
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->update('ngb_targets')
|
||||
->set('schedule', $qb->createNamedParameter(Schedule::validTime($time) ? LocationSchedule::normalize($time) : ''))
|
||||
->set('schedule_zone', $qb->createNamedParameter($zone))
|
||||
->where($qb->expr()->isNull('schedule'));
|
||||
$qb->executeStatement();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Migration;
|
||||
|
||||
use Closure;
|
||||
use OCA\NgBackup\Service\KeyService;
|
||||
use OCP\Migration\IOutput;
|
||||
use OCP\Migration\SimpleMigrationStep;
|
||||
|
||||
/**
|
||||
* A key from before 1.1.0 counts as used: those versions did not keep track, and it may protect
|
||||
* backups on a location that was removed from this server later (user exports included). Only a
|
||||
* key made with 1.1.0 or newer can be replaced as an unused one.
|
||||
*/
|
||||
final class Version000009Date20261108000000 extends SimpleMigrationStep {
|
||||
public function __construct(
|
||||
private KeyService $keys,
|
||||
) {
|
||||
}
|
||||
|
||||
public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
|
||||
if ($this->keys->isInitialized()) {
|
||||
$this->keys->markUsed();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,10 @@ final class Notifier implements INotifier {
|
||||
$notification->setParsedSubject($l->t('Backup to %1$s finished with warnings: %2$s unreadable, %3$s changed while being read', [$p['target'], $p['unreadable'], $p['changing']]));
|
||||
$notification->setParsedMessage($l->t('Unreadable files and folders are missing from this backup (check their permissions). Files that kept changing were saved as they were while being read. Examples: %s', [$p['examples'] ?? '']));
|
||||
break;
|
||||
case 'run_bad_names':
|
||||
$notification->setParsedSubject($l->n('Backup to %s finished, but %n item is missing because of its name', 'Backup to %s finished, but %n items are missing because of their names', (int)$p['count'], [$p['target']]));
|
||||
$notification->setParsedMessage($l->t('Names with a backslash or a control character, or that are not valid UTF-8, are not supported, so these files and folders are not in this backup. Rename them; the next backup includes them. Examples: %s', [$p['examples'] ?? '']));
|
||||
break;
|
||||
case 'quota_warning':
|
||||
$notification->setParsedSubject($l->t('Backup location %1$s is at %2$s%% of its limit (%3$s of %4$s)', [$p['target'], $p['percent'], $p['used'], $p['limit']]));
|
||||
$notification->setParsedMessage($l->t('Backups stop when the limit is reached. Raise the limit, or let the retention policy remove older backups.'));
|
||||
@@ -57,13 +61,21 @@ final class Notifier implements INotifier {
|
||||
: $l->t('No successful backup to %1$s since %2$s', [$p['target'], $p['since']]));
|
||||
$notification->setParsedMessage($l->t('Check the backup settings and the server log.'));
|
||||
break;
|
||||
case 'schedule_off':
|
||||
$notification->setParsedSubject($l->t('Backup location %s has no schedule', [$p['target']]));
|
||||
$notification->setParsedMessage(($p['others'] ?? '') === '1'
|
||||
? $l->t('No backups run to this location until it gets a schedule or someone starts one. If that is what you want, you can ignore this reminder.')
|
||||
: $l->t('No location has a schedule, so no backups run at all unless someone starts one. Set a schedule under Backups.'));
|
||||
break;
|
||||
case 'security_event':
|
||||
$who = (string)($p['actor'] ?? '');
|
||||
$notification->setParsedSubject(match ($p['event'] ?? '') {
|
||||
'kit_downloaded' => $l->t('%s downloaded the backup recovery kit', [$who]),
|
||||
'key_replaced' => $l->t('%1$s replaced the unused backup key %2$s by the key of existing backups, %3$s; download and confirm the new recovery kit', [$who, $p['old'] ?? '', $p['new'] ?? '']),
|
||||
'target_removed' => $l->t('%1$s removed backup location %2$s', [$who, $p['target'] ?? '']),
|
||||
'target_changed' => $l->t('%1$s changed the connection settings of backup location %2$s', [$who, $p['target'] ?? '']),
|
||||
'append_only_off' => $l->t('%1$s turned off append-only for backup location %2$s', [$who, $p['target'] ?? '']),
|
||||
'schedule_off' => $l->t('%1$s turned off the schedule of backup location %2$s', [$who, $p['target'] ?? '']),
|
||||
'append_only_off' =>$l->t('%1$s turned off append-only for backup location %2$s', [$who, $p['target'] ?? '']),
|
||||
'retention_changed' => $l->t('%1$s changed the backup retention to %2$s', [$who, $p['policy'] ?? '']),
|
||||
'snapshots_forgotten' => $l->t('%1$s moved %2$s snapshots of %3$s to the trash; they are deleted permanently after %4$s unless restored', [$who, $p['count'] ?? '', $p['target'] ?? '', $p['until'] ?? '']),
|
||||
'snapshot_untrashed' => $l->t('%1$s restored snapshot %2$s of %3$s from the trash', [$who, $p['snapshot'] ?? '', $p['target'] ?? '']),
|
||||
|
||||
@@ -7,6 +7,7 @@ declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
@@ -32,6 +33,8 @@ final class BlobIndex {
|
||||
private const CACHE_LOT = 256;
|
||||
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
|
||||
public int $downloaded = 0;
|
||||
/** @var array<string, true> packs whose index file is loaded (refresh() fetches only others) */
|
||||
private array $loaded = [];
|
||||
|
||||
public function __construct(
|
||||
private IBackend $backend,
|
||||
@@ -90,13 +93,14 @@ final class BlobIndex {
|
||||
* longer exist on the backend are dropped from the cache.
|
||||
*/
|
||||
public function load(bool $verify = false, ?callable $progress = null): void {
|
||||
$this->blobs = [];
|
||||
$hadIndex = $this->blobs !== [];
|
||||
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
|
||||
$present = [];
|
||||
$this->downloaded = 0;
|
||||
$order = []; // pack ids in listing order: where a blob is in two packs, the last listed wins
|
||||
$fetch = [];
|
||||
foreach ($this->backend->listFiles('index') as $path) {
|
||||
$known = array_keys($this->loaded + $cached + ($verify ? ($this->cache?->all($this->repositoryId) ?? []) : []));
|
||||
foreach ($this->listIndex($known) as $path) {
|
||||
$packId = basename($path);
|
||||
$present[$packId] = true;
|
||||
$order[] = $packId;
|
||||
@@ -104,6 +108,9 @@ final class BlobIndex {
|
||||
$fetch[] = $path;
|
||||
}
|
||||
}
|
||||
// only now: an empty listing that listIndex() refused leaves the index as it was
|
||||
$this->blobs = [];
|
||||
$this->loaded = $present;
|
||||
// The rest is downloaded several at a time where the location allows it: on a large
|
||||
// repository these are tens of thousands of small files. getMany() keeps their order, so
|
||||
// the cached packs listed before each one are added first.
|
||||
@@ -185,12 +192,80 @@ final class BlobIndex {
|
||||
rewind($stream);
|
||||
$this->backend->put('index/' . $packId, $stream);
|
||||
$this->cachePut($packId, $entries);
|
||||
$this->loaded[$packId] = true;
|
||||
foreach ($entries as $e) {
|
||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||
unset($this->pending[$e['id']]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the index files that appeared on the location since load(), such as those of packs
|
||||
* another server's cleanup wrote while a restore runs. Only adds and updates: a blob is
|
||||
* never dropped, as a listing that misses files (a connection that failed) must not make a
|
||||
* restore lose blobs it knows. A blob in a new pack takes that pack's place.
|
||||
*/
|
||||
public function refresh(): void {
|
||||
$fetch = [];
|
||||
$listed = 0;
|
||||
foreach ($this->listIndex(array_keys($this->loaded)) as $path) {
|
||||
$listed++;
|
||||
if (!isset($this->loaded[basename($path)])) {
|
||||
$fetch[] = $path;
|
||||
}
|
||||
}
|
||||
$toCache = [];
|
||||
foreach ($this->backend->getMany($fetch) as $path => $data) {
|
||||
$packId = basename($path);
|
||||
/** @var list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries authenticated: written by NG Backup */
|
||||
$entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
||||
$this->downloaded++;
|
||||
$this->add($packId, $entries);
|
||||
$this->loaded[$packId] = true;
|
||||
$toCache[$packId] = $entries;
|
||||
}
|
||||
if ($toCache !== []) {
|
||||
$this->cache?->putMany($this->repositoryId, $toCache);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The index files on the location. An empty listing while index files are known ($known,
|
||||
* pack ids) is usually a connection that failed quietly; taken as is, it would empty the
|
||||
* index and the cache, make a check report every blob missing, or make a cleanup wipe the
|
||||
* cache. So the connection is checked (the repository's config must be there) and the folder
|
||||
* listed once more. Empty again, three of the known index files are looked up one by one:
|
||||
* only when they are gone too is the repository taken as empty (another server removed its
|
||||
* last snapshot and cleaned up). Even when that is wrong, nothing on the location is lost:
|
||||
* only this index and the cache are empty until the next load.
|
||||
*
|
||||
* @param list<string> $known
|
||||
* @return list<string>
|
||||
* @throws BackendException when the location does not answer as it should
|
||||
*/
|
||||
private function listIndex(array $known): array {
|
||||
$paths = $this->backend->listFiles('index');
|
||||
if ($paths !== [] || $known === []) {
|
||||
return $paths;
|
||||
}
|
||||
if (!$this->backend->exists('config')) {
|
||||
throw new BackendException('The location listed no index files and has no repository config: not trusting that listing');
|
||||
}
|
||||
$paths = $this->backend->listFiles('index');
|
||||
if ($paths !== []) {
|
||||
return $paths;
|
||||
}
|
||||
// spread over the list: first, middle and last (another server's cleanup removes packs
|
||||
// here and there, not in this order)
|
||||
$n = count($known);
|
||||
foreach (array_unique([$known[0], $known[intdiv($n, 2)], $known[$n - 1]]) as $packId) {
|
||||
if ($this->backend->exists('index/' . $packId)) {
|
||||
throw new BackendException('The location listed no index files, while index/' . $packId . ' is there: not trusting that listing');
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/** @return array<string, list<array{id:string, offset:int, length:int}>> packId => blobs currently located in that pack */
|
||||
public function packs(): array {
|
||||
$packs = [];
|
||||
@@ -204,6 +279,7 @@ final class BlobIndex {
|
||||
public function removePack(string $packId): void {
|
||||
$this->backend->delete('index/' . $packId);
|
||||
$this->cacheRemove([$packId]);
|
||||
unset($this->loaded[$packId]);
|
||||
foreach ($this->blobs as $id => $loc) {
|
||||
if ($loc[0] === $packId) {
|
||||
unset($this->blobs[$id]);
|
||||
|
||||
@@ -30,6 +30,10 @@ final class Catalog {
|
||||
public array $trash = [];
|
||||
/** @var array<string, true> manifest path => true, for a per-user export (users/<uid>/<exportId>) */
|
||||
public array $userExports = [];
|
||||
/** @var array<string, true> snapshots the retention policy leaves alone */
|
||||
public array $pinned = [];
|
||||
/** @var array<string, string> snapshot id => note, also while the snapshot is in the trash */
|
||||
public array $notes = [];
|
||||
public string $hash = '';
|
||||
|
||||
public function __construct(
|
||||
@@ -82,6 +86,8 @@ final class Catalog {
|
||||
// Catalogs written before user exports existed have no such key: default to empty rather
|
||||
// than adopting the location's current users/ listing (that would skip verification).
|
||||
$this->userExports = array_fill_keys($data['userExports'] ?? [], true);
|
||||
$this->pinned = array_fill_keys($data['pinned'] ?? [], true);
|
||||
$this->notes = $data['notes'] ?? [];
|
||||
$this->anchor?->set($this->repositoryId, $this->gen, $this->hash);
|
||||
}
|
||||
|
||||
@@ -93,9 +99,12 @@ final class Catalog {
|
||||
if ($this->backend->exists($this->path($next))) {
|
||||
throw new RepositoryException("Catalog generation $next was written by another process at the same time; try again");
|
||||
}
|
||||
// A pin ends when its snapshot leaves the list, a note when the snapshot is gone for good.
|
||||
$this->pinned = array_intersect_key($this->pinned, $this->snapshots);
|
||||
$this->notes = array_intersect_key($this->notes, $this->snapshots + $this->trash);
|
||||
$plain = json_encode(['repository' => $this->repositoryId, 'gen' => $next, 'time' => gmdate('c'), 'prev' => $this->hash,
|
||||
'snapshots' => array_keys($this->snapshots), 'trash' => array_keys($this->trash),
|
||||
'userExports' => array_keys($this->userExports)], JSON_THROW_ON_ERROR);
|
||||
'userExports' => array_keys($this->userExports), 'pinned' => array_keys($this->pinned), 'notes' => $this->notes], JSON_THROW_ON_ERROR);
|
||||
$encrypted = $this->cipher->encryptString($plain, $this->ad($next));
|
||||
$stream = fopen('php://memory', 'w+b');
|
||||
fwrite($stream, $encrypted);
|
||||
|
||||
@@ -8,9 +8,10 @@ declare(strict_types=1);
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Backend\IReadsAhead;
|
||||
|
||||
/** The location as Repository sees it: closes the PackReader's open stream before every other request. */
|
||||
final class PackReaderGuard implements IBackend {
|
||||
final class PackReaderGuard implements IBackend, IReadsAhead {
|
||||
public function __construct(
|
||||
private IBackend $inner,
|
||||
private PackReader $reader,
|
||||
@@ -43,6 +44,15 @@ final class PackReaderGuard implements IBackend {
|
||||
return $this->inner->getRanges($ranges);
|
||||
}
|
||||
|
||||
public function canReadAhead(): bool {
|
||||
return $this->inner instanceof IReadsAhead && $this->inner->canReadAhead();
|
||||
}
|
||||
|
||||
/** Over connections of the location's own: the open stream may stay open. */
|
||||
public function readAhead(array $ranges, string $spool): ?\Generator {
|
||||
return $this->inner instanceof IReadsAhead ? $this->inner->readAhead($ranges, $spool) : null;
|
||||
}
|
||||
|
||||
public function exists(string $path): bool {
|
||||
$this->reader->close();
|
||||
return $this->inner->exists($path);
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
/**
|
||||
* The ranges of packs a restore reads, in order, for IReadsAhead::readAhead(), and the reading of
|
||||
* blobs from them as the restore goes.
|
||||
*
|
||||
* add() takes the blobs in the order the restore needs them. A blob in the same pack as the range
|
||||
* before it, not before that range and at most GAP after its end, extends that range; anything
|
||||
* else starts a new one. read() takes the same blobs in the same order: a blob is in the current
|
||||
* range or in the next one, which then becomes current; a blob not in the plan is read elsewhere.
|
||||
*/
|
||||
final class ReadAheadPlan {
|
||||
/** Unused bytes between two blobs of a pack that are read along rather than asked for apart. */
|
||||
public const GAP = 4 * 1048576;
|
||||
/** Longest range: packs are 32 MiB, repacked ones too. */
|
||||
public const MAX_RANGE = 64 * 1048576;
|
||||
/** Ranges after the current one that read() looks in for a blob. */
|
||||
private const SEARCH = 16;
|
||||
|
||||
/** @var list<array{0:string, 1:int, 2:int}> [path, offset, length] */
|
||||
private array $ranges = [];
|
||||
/** @var \Generator<int, resource|null>|null */
|
||||
private ?\Generator $streams = null;
|
||||
private int $current = -1;
|
||||
/** @var resource|null */
|
||||
private $stream = null;
|
||||
|
||||
/** @param string $spool local folder for the ranges read ahead (see IReadsAhead::readAhead()) */
|
||||
public function __construct(
|
||||
public readonly string $spool,
|
||||
) {
|
||||
}
|
||||
|
||||
public function add(string $path, int $offset, int $length): void {
|
||||
$last = array_pop($this->ranges);
|
||||
if ($last !== null) {
|
||||
[$p, $start, $len] = $last;
|
||||
if ($p === $path && $offset >= $start && $offset <= $start + $len + self::GAP
|
||||
&& $offset + $length - $start <= self::MAX_RANGE) {
|
||||
$this->ranges[] = [$p, $start, max($len, $offset + $length - $start)];
|
||||
return;
|
||||
}
|
||||
$this->ranges[] = $last;
|
||||
}
|
||||
$this->ranges[] = [$path, $offset, $length];
|
||||
}
|
||||
|
||||
/** @return list<array{0:string, 1:int, 2:int}> */
|
||||
public function ranges(): array {
|
||||
return $this->ranges;
|
||||
}
|
||||
|
||||
/** @param \Generator<int, resource|null> $streams what IReadsAhead::readAhead() gave for ranges() */
|
||||
public function readFrom(\Generator $streams): void {
|
||||
$this->streams = $streams;
|
||||
}
|
||||
|
||||
/** The bytes of a blob, or null when its range could not be read ahead (read it the usual way). */
|
||||
public function read(string $path, int $offset, int $length): ?string {
|
||||
if ($this->streams === null) {
|
||||
return null;
|
||||
}
|
||||
if (!$this->contains($this->current, $path, $offset, $length)) {
|
||||
// Normally the next range. Further on when blobs before it were read elsewhere: the
|
||||
// index was reloaded after another server's cleanup moved them to new packs.
|
||||
$to = null;
|
||||
for ($i = $this->current + 1; $i <= $this->current + self::SEARCH && $to === null; $i++) {
|
||||
if ($this->contains($i, $path, $offset, $length)) {
|
||||
$to = $i;
|
||||
}
|
||||
}
|
||||
if ($to === null) {
|
||||
return null; // not where the plan has it
|
||||
}
|
||||
if ($this->current < 0) {
|
||||
$this->streams->current();
|
||||
$this->current = 0;
|
||||
}
|
||||
while ($this->current < $to) {
|
||||
$this->streams->next();
|
||||
$this->current++;
|
||||
}
|
||||
$stream = $this->streams->valid() ? $this->streams->current() : null;
|
||||
$this->stream = is_resource($stream) ? $stream : null;
|
||||
}
|
||||
$fh = $this->stream;
|
||||
if ($fh === null || fseek($fh, $offset - $this->ranges[$this->current][1]) !== 0) {
|
||||
return null;
|
||||
}
|
||||
$data = '';
|
||||
while (strlen($data) < $length) {
|
||||
$chunk = fread($fh, $length - strlen($data));
|
||||
if ($chunk === false || $chunk === '') {
|
||||
return null;
|
||||
}
|
||||
$data .= $chunk;
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
/** Ends the reads ahead, also halfway: closes their streams and connections. */
|
||||
public function close(): void {
|
||||
$this->stream = null;
|
||||
$streams = $this->streams;
|
||||
$this->streams = null;
|
||||
unset($streams); // the generator's finally blocks run when it is destroyed
|
||||
}
|
||||
|
||||
private function contains(int $i, string $path, int $offset, int $length): bool {
|
||||
if ($i < 0 || !isset($this->ranges[$i])) {
|
||||
return false;
|
||||
}
|
||||
[$p, $start, $len] = $this->ranges[$i];
|
||||
return $p === $path && $offset >= $start && $offset + $length <= $start + $len;
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ namespace OCA\NgBackup\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Backend\IReadsAhead;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Crypto\StreamCipher;
|
||||
|
||||
@@ -17,6 +18,7 @@ use OCA\NgBackup\Crypto\StreamCipher;
|
||||
*
|
||||
* Layout on the backend:
|
||||
* config plaintext JSON: format version, repository id, wrapped master key
|
||||
* settings encrypted JSON: settings fixed at installation (deletion delay)
|
||||
* packs/xx/<packId> encrypted blobs + encrypted pack header
|
||||
* index/<packId> encrypted list of the blobs in that pack
|
||||
* snapshots/<snapId> encrypted JSON lines: one meta line, then one line per file
|
||||
@@ -27,6 +29,7 @@ use OCA\NgBackup\Crypto\StreamCipher;
|
||||
final class Repository {
|
||||
public const FORMAT = 1;
|
||||
public const BLOB_SIZE = 4 * 1048576;
|
||||
private const SETTINGS = 'settings';
|
||||
private const FLAG_DEFLATE = 1;
|
||||
/** Blobs from this size on get a quick compression test before the full compression. */
|
||||
private const DEFLATE_PROBE_FROM = 256 * 1024;
|
||||
@@ -43,6 +46,8 @@ final class Repository {
|
||||
private ?int $maxBytes = null;
|
||||
private Catalog $catalog;
|
||||
private PackReader $packReader;
|
||||
/** @var array<string, array<string, mixed>> meta record of the snapshot read last */
|
||||
private array $lastMeta = [];
|
||||
/** verify() lists the pack folders from this many packs on (see presentPacks()); tests lower it. */
|
||||
public int $listPacksAbove = 256;
|
||||
|
||||
@@ -71,6 +76,14 @@ final class Repository {
|
||||
if ($backend->exists('config')) {
|
||||
throw new RepositoryException('A repository already exists at this location');
|
||||
}
|
||||
// No config, but backup data: the config is missing (or could not be seen just now, a
|
||||
// connection that failed). A new repository here would write a config over the one that
|
||||
// opens this data; refuse instead.
|
||||
foreach (['index', 'snapshots', 'trash/snapshots', 'catalog'] as $dir) {
|
||||
if ($backend->listFiles($dir) !== []) {
|
||||
throw new RepositoryException("This location holds backup data ($dir/) but no repository config could be read: not creating a new repository over it. Check the connection and the folder. If the config is really gone, occ backup:target:add with --repair-config writes it again with this server's key (on a new server: import the recovery kit first, occ backup:key:import).");
|
||||
}
|
||||
}
|
||||
$id = bin2hex(random_bytes(16));
|
||||
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
|
||||
'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))];
|
||||
@@ -81,21 +94,17 @@ final class Repository {
|
||||
}
|
||||
|
||||
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
|
||||
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null): self {
|
||||
/** @param ?callable(int, int): void $progress index files downloaded so far, and in all (see BlobIndex::load()) */
|
||||
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null, ?callable $progress = null): self {
|
||||
$config = self::readConfig($backend);
|
||||
try {
|
||||
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
|
||||
} catch (\OCA\NgBackup\Crypto\CryptoException) {
|
||||
$ok = false;
|
||||
}
|
||||
if (!$ok) {
|
||||
if (!self::configMatches($config, $keys)) {
|
||||
throw new RepositoryException('This repository was created with a different key');
|
||||
}
|
||||
$repo = new self($backend, $keys, $config['id']);
|
||||
if ($cache !== null) {
|
||||
$repo->index->setCache($cache, $config['id']);
|
||||
}
|
||||
$repo->index->load();
|
||||
$repo->index->load(false, $progress);
|
||||
$repo->loadCatalog($anchor);
|
||||
return $repo;
|
||||
}
|
||||
@@ -122,6 +131,49 @@ final class Repository {
|
||||
return $this->catalog->gen;
|
||||
}
|
||||
|
||||
/** @return list<string> snapshots the retention policy leaves alone */
|
||||
public function pinnedSnapshots(): array {
|
||||
return array_map('strval', array_keys($this->catalog->pinned));
|
||||
}
|
||||
|
||||
/** @return array<string, string> snapshot id => note */
|
||||
public function snapshotNotes(): array {
|
||||
return $this->catalog->notes;
|
||||
}
|
||||
|
||||
public function setPinned(string $snapshotId, bool $pinned): void {
|
||||
$this->requireSnapshot($snapshotId);
|
||||
if (isset($this->catalog->pinned[$snapshotId]) === $pinned) {
|
||||
return;
|
||||
}
|
||||
if ($pinned) {
|
||||
$this->catalog->pinned[$snapshotId] = true;
|
||||
} else {
|
||||
unset($this->catalog->pinned[$snapshotId]);
|
||||
}
|
||||
$this->catalog->write();
|
||||
}
|
||||
|
||||
/** A note kept in the catalog: unlike the label inside the snapshot, it can change afterwards. */
|
||||
public function setSnapshotNote(string $snapshotId, string $note): void {
|
||||
$this->requireSnapshot($snapshotId);
|
||||
if (($this->catalog->notes[$snapshotId] ?? '') === $note) {
|
||||
return;
|
||||
}
|
||||
if ($note === '') {
|
||||
unset($this->catalog->notes[$snapshotId]);
|
||||
} else {
|
||||
$this->catalog->notes[$snapshotId] = $note;
|
||||
}
|
||||
$this->catalog->write();
|
||||
}
|
||||
|
||||
private function requireSnapshot(string $snapshotId): void {
|
||||
if (!isset($this->catalog->snapshots[$snapshotId])) {
|
||||
throw new \InvalidArgumentException('There is no snapshot ' . $snapshotId . ' on this location');
|
||||
}
|
||||
}
|
||||
|
||||
/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
|
||||
public function indexDownloads(): int {
|
||||
return $this->index->downloaded;
|
||||
@@ -141,6 +193,114 @@ final class Repository {
|
||||
return $config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a local file to read it in whole blobs. PHP reads a file in pieces of 8 KiB by default;
|
||||
* on a disk where each request waits (a network or cloud volume) that made reading five times
|
||||
* slower: 5 MiB/s instead of 27 MiB/s on nc01. Without PHP's read buffer every fread() of a
|
||||
* blob is one request to the disk.
|
||||
*
|
||||
* @return resource|false
|
||||
*/
|
||||
public static function openForReading(string $path) {
|
||||
$fh = @fopen($path, 'rb');
|
||||
if ($fh !== false) {
|
||||
stream_set_read_buffer($fh, 0);
|
||||
stream_set_chunk_size($fh, self::BLOB_SIZE);
|
||||
}
|
||||
return $fh;
|
||||
}
|
||||
|
||||
private static function configMatches(array $config, KeyRing $keys): bool {
|
||||
try {
|
||||
return isset($config['check'], $config['id'])
|
||||
&& (new StreamCipher($keys))->decryptString((string)base64_decode((string)$config['check'], true), 'config-check') === $config['id'];
|
||||
} catch (\OCA\NgBackup\Crypto\CryptoException) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The passphrase-wrapped key slots in the config at this location: disaster recovery without
|
||||
* the recovery kit, where any one of their passphrases gives back the master key.
|
||||
*/
|
||||
public static function wrappedKeyAt(IBackend $backend): array {
|
||||
$config = self::readConfig($backend);
|
||||
if (!isset($config['key']) || !is_array($config['key'])) {
|
||||
throw new RepositoryException('The repository config at this location holds no key');
|
||||
}
|
||||
return $config['key'];
|
||||
}
|
||||
|
||||
/** Whether the repository at this location was created with this key. Does not read the index. */
|
||||
public static function madeWithKey(IBackend $backend, KeyRing $keys): bool {
|
||||
return self::configMatches(self::readConfig($backend), $keys);
|
||||
}
|
||||
|
||||
/**
|
||||
* Settings of the installation that never change (the deletion delay), sealed with the
|
||||
* master key so that, unlike the config and the recovery kit, nobody can alter them unseen.
|
||||
* Empty when the location has none yet (made before 1.1.0, until its next backup); only
|
||||
* ['unreadable' => true] when they are there but cannot be read.
|
||||
*
|
||||
* @return array<string, mixed>
|
||||
*/
|
||||
public static function settingsAt(IBackend $backend, KeyRing $keys): array {
|
||||
try {
|
||||
if (!$backend->exists(self::SETTINGS)) {
|
||||
return [];
|
||||
}
|
||||
$fh = $backend->get(self::SETTINGS);
|
||||
$raw = (string)stream_get_contents($fh);
|
||||
fclose($fh);
|
||||
$settings = json_decode((new StreamCipher($keys))->decryptString($raw, 'settings'), true, 16, JSON_THROW_ON_ERROR);
|
||||
return is_array($settings) ? $settings : ['unreadable' => true];
|
||||
} catch (BackendException|\OCA\NgBackup\Crypto\CryptoException|\JsonException) {
|
||||
return ['unreadable' => true];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the settings once; a location that has them keeps them. The repository id goes with
|
||||
* them: with it and the key from the recovery kit, a lost config can be written again.
|
||||
*/
|
||||
public function recordSettings(array $settings): void {
|
||||
if ($this->backend->exists(self::SETTINGS)) {
|
||||
return;
|
||||
}
|
||||
$settings['repository'] = $this->repositoryId;
|
||||
self::putString($this->backend, self::SETTINGS, $this->cipher->encryptString(json_encode($settings, JSON_THROW_ON_ERROR), 'settings'));
|
||||
}
|
||||
|
||||
/** Whether the location holds backup data, with or without a config. */
|
||||
public static function holdsData(IBackend $backend): bool {
|
||||
foreach (['index', 'snapshots', 'catalog'] as $dir) {
|
||||
if ($backend->listFiles($dir) !== []) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a lost config again, from the key (recovery kit) and the repository id in the
|
||||
* sealed settings. Only where there is no config and the settings open with this key, so
|
||||
* nothing that works is ever overwritten and a wrong key writes nothing.
|
||||
*/
|
||||
public static function repairConfig(IBackend $backend, KeyRing $keys, array $wrappedKey): void {
|
||||
if ($backend->exists('config')) {
|
||||
throw new RepositoryException('This location has a config: nothing to repair, leave out --repair-config');
|
||||
}
|
||||
$id = self::settingsAt($backend, $keys)['repository'] ?? null;
|
||||
if (!is_string($id) || !preg_match('/^[0-9a-f]{32}$/', $id)) {
|
||||
throw new RepositoryException($backend->exists(self::SETTINGS)
|
||||
? 'The settings at this location do not open with this key: the recovery kit is of another server, or the settings are damaged. The config cannot be written again.'
|
||||
: 'This location has no settings file (it was backed up only by NG Backup before 1.1.0), so its config cannot be written again.');
|
||||
}
|
||||
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
|
||||
'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check')), 'repaired' => true];
|
||||
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
||||
}
|
||||
|
||||
/** Replace the passphrase-wrapped key slots in this repository's config (after a slot change). */
|
||||
public function updateWrappedKey(array $wrappedKey): void {
|
||||
$config = self::readConfig($this->backend);
|
||||
@@ -204,63 +364,336 @@ final class Repository {
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a snapshot (or only paths starting with $prefix) into a local directory.
|
||||
* Restore a snapshot (or only paths starting with $prefix) into a local directory. A $prefix
|
||||
* "dir/file/" also restores the file "dir/file" itself.
|
||||
*
|
||||
* @param bool $stripPrefix write "$target/<path without $prefix>" instead of
|
||||
* "$target/<full logical path>" (disaster recovery: restoring e.g. "data/" or
|
||||
* "config/" directly into the real data/config directory, not a subdirectory
|
||||
* named after the prefix). Ignored when $prefix is '' (nothing to strip).
|
||||
* @param array<string, string>|null $skipped salvage: an array to collect the files that could
|
||||
* not be read or decrypted (path => reason), which are then left out instead of
|
||||
* stopping the restore; null (default) stops at the first such file
|
||||
* @param bool $keepUnchanged leave a file that is already in place with the size and
|
||||
* modification time of the snapshot, as a backup takes such a file as unchanged:
|
||||
* a restore that was stopped and runs again goes on where it was, without reading
|
||||
* what it already restored from the location again
|
||||
*/
|
||||
public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null): int {
|
||||
public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null, bool $keepUnchanged = false): int {
|
||||
if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) {
|
||||
throw new RepositoryException('Invalid path prefix');
|
||||
}
|
||||
$plan = $this->readAhead($snapshotId, $prefix, $target, $stripPrefix, $keepUnchanged);
|
||||
try {
|
||||
return $this->restoreEntries($snapshotId, $target, $prefix, $stripPrefix, $heartbeat, $plan, $skipped, $keepUnchanged);
|
||||
} finally {
|
||||
$plan?->close();
|
||||
if ($plan !== null) {
|
||||
@rmdir($plan->spool);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start reading the packs of a restore ahead, when the location can (IReadsAhead): a first
|
||||
* pass over the snapshot lists the ranges of packs it needs, in order. The spool is a
|
||||
* folder in the target, which has room for the restore anyway.
|
||||
*/
|
||||
private function readAhead(string $snapshotId, string $prefix, string $target, bool $stripPrefix, bool $keepUnchanged): ?ReadAheadPlan {
|
||||
if (!$this->backend instanceof IReadsAhead || !$this->backend->canReadAhead()) {
|
||||
return null;
|
||||
}
|
||||
// what a restore that was stopped left behind: its files are unlinked once opened, but one
|
||||
// that was being written when the process was killed stays (8 MiB, DR test 2026-11-05)
|
||||
$stale = glob(rtrim($target, '/') . '/.ngb-restore-spool-*', GLOB_ONLYDIR);
|
||||
foreach ($stale === false ? [] : $stale as $dir) {
|
||||
if (is_link($dir) || preg_match('/^\.ngb-restore-spool-[0-9a-f]{8}$/', basename($dir)) !== 1) {
|
||||
continue; // not a spool of ours, whatever it is called
|
||||
}
|
||||
$files = glob($dir . '/*');
|
||||
foreach ($files === false ? [] : $files as $file) {
|
||||
if (is_file($file) && !is_link($file)) {
|
||||
@unlink($file);
|
||||
}
|
||||
}
|
||||
@rmdir($dir);
|
||||
}
|
||||
$plan = new ReadAheadPlan(rtrim($target, '/') . '/.ngb-restore-spool-' . bin2hex(random_bytes(4)));
|
||||
try {
|
||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||
if (!self::underPrefix($entry['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
if ($keepUnchanged && self::inPlace(self::destination($target, $entry['p'], $prefix, $stripPrefix), $entry)) {
|
||||
continue;
|
||||
}
|
||||
foreach ($entry['b'] as $id) {
|
||||
try {
|
||||
[$pack, $offset, $length] = $this->index->get($id);
|
||||
} catch (RepositoryException) {
|
||||
continue; // the restore reports it when it gets there
|
||||
}
|
||||
$plan->add('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||
}
|
||||
}
|
||||
} catch (\Throwable) {
|
||||
return null; // reading ahead is only faster: the restore itself reads the snapshot again and reports what is wrong
|
||||
}
|
||||
if (count($plan->ranges()) < 2 || (!is_dir($plan->spool) && !@mkdir($plan->spool, 0700, true))) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
$streams = $this->backend->readAhead($plan->ranges(), $plan->spool);
|
||||
} catch (\Throwable) {
|
||||
$streams = null; // the usual reads report what is wrong
|
||||
}
|
||||
if ($streams === null) {
|
||||
@rmdir($plan->spool);
|
||||
return null;
|
||||
}
|
||||
$plan->readFrom($streams);
|
||||
return $plan;
|
||||
}
|
||||
|
||||
/** @param array<string, string>|null $skipped see restore() */
|
||||
private function restoreEntries(string $snapshotId, string $target, string $prefix, bool $stripPrefix, ?callable $heartbeat, ?ReadAheadPlan $plan, ?array &$skipped, bool $keepUnchanged): int {
|
||||
$count = 0;
|
||||
$cleaned = [];
|
||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||
if (!self::underPrefix($entry['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
$relPath = ($stripPrefix && $prefix !== '') ? substr($entry['p'], strlen(rtrim($prefix, '/')) + 1) : $entry['p'];
|
||||
$dest = rtrim($target, '/') . '/' . $relPath;
|
||||
if (!is_dir(dirname($dest))) {
|
||||
mkdir(dirname($dest), 0700, true);
|
||||
if ($heartbeat !== null) {
|
||||
$heartbeat(); // also for a long row of files that are skipped or have no data
|
||||
}
|
||||
// Defence in depth: the resolved directory must stay below the target (no symlink escapes).
|
||||
$base = realpath($target);
|
||||
$dir = realpath(dirname($dest));
|
||||
if ($base === false || $dir === false || ($dir !== $base && !str_starts_with($dir, $base . '/')) || is_link($dest)) {
|
||||
throw new RepositoryException('Refusing to write outside the restore directory: ' . $entry['p']);
|
||||
$dest = self::destination($target, $entry['p'], $prefix, $stripPrefix);
|
||||
$dir = self::directoryInTarget($target, dirname($dest), $entry['p']);
|
||||
if (is_link($dest)) {
|
||||
throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $entry['p']);
|
||||
}
|
||||
if (!isset($cleaned[$dir])) {
|
||||
self::removeStaleTemporaries($dir);
|
||||
$cleaned[$dir] = true;
|
||||
}
|
||||
if ($keepUnchanged && self::inPlace($dest, $entry)) {
|
||||
$count++;
|
||||
continue;
|
||||
}
|
||||
// Write next to the destination and rename after all blobs verified: at most one file
|
||||
// is ever present twice.
|
||||
$tmp = $dest . '.ngb-restore-' . bin2hex(random_bytes(6));
|
||||
$out = @fopen($tmp, 'xb'); // fails if anything (including a symlink) already exists there
|
||||
if ($out === false) {
|
||||
throw new RepositoryException('Cannot create a temporary file for ' . $entry['p']);
|
||||
throw new RestoreTargetException('Cannot create a temporary file for ' . $entry['p'] . ' (disk full, or no permission?)');
|
||||
}
|
||||
foreach ($entry['b'] as $id) {
|
||||
fwrite($out, $this->loadBlob($id));
|
||||
if ($heartbeat !== null) {
|
||||
$heartbeat();
|
||||
try {
|
||||
foreach ($entry['b'] as $id) {
|
||||
self::writeAll($out, $this->restoreBlob($id, $plan), $entry['p']);
|
||||
if ($heartbeat !== null) {
|
||||
$heartbeat();
|
||||
}
|
||||
}
|
||||
}
|
||||
fclose($out);
|
||||
if (filesize($tmp) !== $entry['s']) {
|
||||
unlink($tmp);
|
||||
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
||||
if (!@fclose($out)) {
|
||||
throw new RestoreTargetException('Cannot write ' . $entry['p'] . ' to the restore directory (disk full?)');
|
||||
}
|
||||
clearstatcache(true, $tmp);
|
||||
if (filesize($tmp) !== $entry['s']) {
|
||||
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
||||
}
|
||||
} catch (RepositoryException|BackendException|\OCA\NgBackup\Crypto\CryptoException $e) {
|
||||
// What the location gave back for this file is missing or damaged. Writing to the
|
||||
// target (below) is not part of this: a full disk must still stop the restore.
|
||||
if (is_resource($out)) {
|
||||
fclose($out);
|
||||
}
|
||||
@unlink($tmp);
|
||||
if ($skipped === null) {
|
||||
throw $e;
|
||||
}
|
||||
$skipped[$entry['p']] = $e->getMessage();
|
||||
continue;
|
||||
} catch (\Throwable $e) {
|
||||
if (is_resource($out)) {
|
||||
@fclose($out);
|
||||
}
|
||||
@unlink($tmp);
|
||||
throw $e;
|
||||
}
|
||||
if (!rename($tmp, $dest)) {
|
||||
@unlink($tmp);
|
||||
throw new RepositoryException('Cannot put the restored file in place: ' . $entry['p']);
|
||||
throw new RestoreTargetException('Cannot put the restored file in place: ' . $entry['p']);
|
||||
}
|
||||
if (!touch($dest, $entry['m'])) {
|
||||
throw new RepositoryException('Cannot set the modification time of ' . $entry['p']);
|
||||
throw new RestoreTargetException('Cannot set the modification time of ' . $entry['p']);
|
||||
}
|
||||
$count++;
|
||||
}
|
||||
// Last: a file can only be restored into a folder that exists, and the meta record was
|
||||
// read along with the files, so this asks nothing more of the location.
|
||||
foreach ($this->emptyDirs($snapshotId) as [$path, $mtime]) {
|
||||
if (self::underPrefix($path, $prefix)) {
|
||||
if ($heartbeat !== null) {
|
||||
$heartbeat();
|
||||
}
|
||||
// The prefix itself, stripped, is the target (for a file it is the file's name there).
|
||||
$dest = $stripPrefix && $path === rtrim($prefix, '/') ? rtrim($target, '/') : self::destination($target, $path, $prefix, $stripPrefix);
|
||||
self::restoreEmptyDir($target, $dest, $path, $mtime);
|
||||
}
|
||||
}
|
||||
// Very last, nothing is written in them any more. Also on a run that goes on after a stop, and
|
||||
// the deepest first. The time is not worth stopping for: all the files are there by now.
|
||||
foreach (array_reverse($this->folderTimes($snapshotId)) as [$path, $mtime]) {
|
||||
if (!self::underPrefix($path, $prefix) || ($stripPrefix && $path === rtrim($prefix, '/'))) {
|
||||
continue; // the target itself is not ours to change
|
||||
}
|
||||
if ($heartbeat !== null) {
|
||||
$heartbeat();
|
||||
}
|
||||
$dest = self::destination($target, $path, $prefix, $stripPrefix);
|
||||
@touch(self::directoryInTarget($target, $dest, $path), $mtime);
|
||||
}
|
||||
return $count;
|
||||
}
|
||||
|
||||
/**
|
||||
* The folders with something in them that a snapshot recorded, parents before children (none
|
||||
* before 1.1.0). Damage here is no reason to give up on the files of a restore.
|
||||
*
|
||||
* @return list<array{0: string, 1: int}> path, modification time
|
||||
*/
|
||||
public function folderTimes(string $snapshotId): array {
|
||||
$meta = $this->lastMeta[$snapshotId] ?? $this->snapshotMeta($snapshotId);
|
||||
$packed = is_string($meta['folderTimes'] ?? null) ? base64_decode($meta['folderTimes'], true) : false;
|
||||
$plain = $packed === false ? false : @gzinflate($packed);
|
||||
$folders = [];
|
||||
foreach ($plain === false ? [] : explode("\n", $plain) as $line) {
|
||||
$folder = json_decode($line, true);
|
||||
if (is_array($folder) && is_string($folder[0] ?? null) && self::isSafePath($folder[0]) && is_int($folder[1] ?? null)) {
|
||||
$folders[] = [$folder[0], $folder[1]];
|
||||
}
|
||||
}
|
||||
return $folders;
|
||||
}
|
||||
|
||||
/**
|
||||
* The folders without anything in them that a snapshot recorded (none before 1.0.3).
|
||||
*
|
||||
* @return list<array{0: string, 1: int}> path, modification time
|
||||
*/
|
||||
public function emptyDirs(string $snapshotId): array {
|
||||
$meta = $this->lastMeta[$snapshotId] ?? $this->snapshotMeta($snapshotId);
|
||||
$dirs = [];
|
||||
foreach (is_array($meta['emptyDirs'] ?? null) ? $meta['emptyDirs'] : [] as $dir) {
|
||||
// One folder that makes no sense is no reason to give up on the files of a restore.
|
||||
if (is_array($dir) && is_string($dir[0] ?? null) && self::isSafePath($dir[0]) && is_int($dir[1] ?? null)) {
|
||||
$dirs[] = [$dir[0], $dir[1]];
|
||||
}
|
||||
}
|
||||
return $dirs;
|
||||
}
|
||||
|
||||
private static function restoreEmptyDir(string $target, string $dest, string $path, int $mtime): void {
|
||||
if (is_link($dest)) {
|
||||
throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $path);
|
||||
}
|
||||
$existed = is_dir($dest);
|
||||
self::directoryInTarget($target, $dest, $path);
|
||||
if (!$existed && !@touch($dest, $mtime)) {
|
||||
throw new RestoreTargetException('Cannot set the modification time of ' . $path);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure a folder exists below the target, without following a link out of it: what exists
|
||||
* of the path is checked before anything is created.
|
||||
*
|
||||
* @return string the folder's real path
|
||||
*/
|
||||
private static function directoryInTarget(string $target, string $dir, string $path): string {
|
||||
if (!is_dir($target) && !@mkdir($target, 0700, true) && !is_dir($target)) {
|
||||
throw new RestoreTargetException('Cannot create the restore directory ' . $target);
|
||||
}
|
||||
$base = realpath($target);
|
||||
$existing = $dir;
|
||||
while (!file_exists($existing) && !is_link($existing) && dirname($existing) !== $existing) {
|
||||
$existing = dirname($existing);
|
||||
}
|
||||
$real = realpath($existing);
|
||||
if ($base === false || $real === false || ($real !== $base && !str_starts_with($real, $base . '/'))) {
|
||||
throw new RestoreTargetException('Refusing to write outside the restore directory: ' . $path);
|
||||
}
|
||||
if ($existing === $dir) {
|
||||
if (!is_dir($real)) {
|
||||
throw new RestoreTargetException('Cannot create the folder for ' . $path . ': a file is in the way');
|
||||
}
|
||||
return $real;
|
||||
}
|
||||
if (!@mkdir($dir, 0700, true) && !is_dir($dir)) {
|
||||
throw new RestoreTargetException('Cannot create the folder for ' . $path . ' (disk full, no permission, or a file in the way?)');
|
||||
}
|
||||
return (string)realpath($dir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write all of $data, or stop: a short write leaves a file that looks damaged only by its size.
|
||||
* A write may take only part of the data and still be fine; one that takes nothing is not.
|
||||
*
|
||||
* @param resource $out
|
||||
*/
|
||||
public static function writeAll($out, string $data, string $path): void {
|
||||
while ($data !== '') {
|
||||
$written = @fwrite($out, $data);
|
||||
if ($written === false || $written === 0) {
|
||||
throw new RestoreTargetException('Cannot write ' . $path . ' to the restore directory (disk full?)');
|
||||
}
|
||||
$data = substr($data, $written);
|
||||
}
|
||||
}
|
||||
|
||||
/** Where restore() writes the snapshot path $path. */
|
||||
private static function destination(string $target, string $path, string $prefix, bool $stripPrefix): string {
|
||||
$relPath = $path;
|
||||
if ($stripPrefix && $prefix !== '') {
|
||||
$relPath = $path === rtrim($prefix, '/') ? basename($path) : substr($path, strlen($prefix));
|
||||
}
|
||||
return rtrim($target, '/') . '/' . $relPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* A regular file (no link) at $dest with the size and modification time of the snapshot
|
||||
* entry: what a backup takes as unchanged, and what a finished restore of it leaves.
|
||||
*
|
||||
* @param array{s: int, m: int, ...} $entry
|
||||
*/
|
||||
private static function inPlace(string $dest, array $entry): bool {
|
||||
clearstatcache(true, $dest);
|
||||
return !is_link($dest) && is_file($dest) && filesize($dest) === $entry['s'] && filemtime($dest) === $entry['m'];
|
||||
}
|
||||
|
||||
/** $prefix is "" (everything) or ends with "/"; the path without that "/" is a file to restore too. */
|
||||
private static function underPrefix(string $path, string $prefix): bool {
|
||||
return $prefix === '' || str_starts_with($path, $prefix) || $path === rtrim($prefix, '/');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove what a restore that was stopped left in a directory this restore writes to:
|
||||
* "<name>.ngb-restore-<hex>" from here, ".<name>.ngb-restore-<hex>" from a restore through
|
||||
* Nextcloud. A files:scan would otherwise make them show up in Files.
|
||||
*/
|
||||
public static function removeStaleTemporaries(string $dir): void {
|
||||
$names = scandir($dir);
|
||||
foreach ($names === false ? [] : $names as $name) {
|
||||
$path = $dir . '/' . $name;
|
||||
if (self::isRestoreTemporary($name) && is_file($path) && !is_link($path)) {
|
||||
@unlink($path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static function isRestoreTemporary(string $name): bool {
|
||||
return preg_match('/.\.ngb-restore-[0-9a-f]{6}(?:[0-9a-f]{6})?$/', $name) === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start writing a generated stream (e.g. a table dump) as blobs. Call flushPacks() when the
|
||||
* run is done, so the last pack is uploaded before anything references it.
|
||||
@@ -491,6 +924,31 @@ final class Repository {
|
||||
return isset($this->catalog->userExports[$manifestPath]);
|
||||
}
|
||||
|
||||
/**
|
||||
* The recorded export $ref names: a manifest path (users/<uid>/<exportId>) as is, or a bare
|
||||
* user id (also "users/<uid>") when that user has exactly one export here. Anything else
|
||||
* throws, with the exports to choose from in the message.
|
||||
*/
|
||||
public function resolveUserExport(string $ref): string {
|
||||
if ($this->isRecordedUserExport($ref)) {
|
||||
return $ref;
|
||||
}
|
||||
$rest = trim($ref, '/');
|
||||
if (str_starts_with($rest, 'users/')) {
|
||||
$rest = substr($rest, strlen('users/'));
|
||||
}
|
||||
$uid = explode('/', $rest)[0];
|
||||
$exports = $uid !== '' ? $this->listUserExports($uid) : [];
|
||||
if (!str_contains($rest, '/') && count($exports) === 1) {
|
||||
return $exports[0];
|
||||
}
|
||||
if ($exports === []) {
|
||||
throw new RepositoryException("$ref is not a recorded user export for this location, and $uid has no exports here. Expected users/<uid>/<exportId>; occ backup:user:list shows the exports");
|
||||
}
|
||||
$which = str_contains($rest, '/') ? "$ref is not a recorded user export for this location" : "$uid has " . count($exports) . ' exports here';
|
||||
throw new RepositoryException("$which; pass one of these (newest first):\n " . implode("\n ", $exports));
|
||||
}
|
||||
|
||||
/** Permanently delete a user export. Its blobs are freed by the next prune() like any other. */
|
||||
public function forgetUserExport(string $manifestPath): void {
|
||||
if (!isset($this->catalog->userExports[$manifestPath])) {
|
||||
@@ -538,6 +996,34 @@ final class Repository {
|
||||
return array_keys($doomed);
|
||||
}
|
||||
|
||||
/**
|
||||
* Part of trash(), for a page that shows how far it is: each entry is a read of its own on the
|
||||
* location. Reads the entries with an id after $after, in order, until $deadline
|
||||
* (microtime(true)), at least one. The id, not a position: a cleanup that changes the trash
|
||||
* between two parts then makes no entry come twice or be missed.
|
||||
*
|
||||
* @return array{total: int, items: array<string, array<string, mixed>>}
|
||||
*/
|
||||
public function trashPage(string $after, float $deadline): array {
|
||||
$ids = array_map('strval', array_keys($this->catalog->trash));
|
||||
sort($ids, SORT_STRING);
|
||||
$items = [];
|
||||
foreach ($ids as $id) {
|
||||
if (strcmp($id, $after) <= 0) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
$items[$id] = $this->trashInfo($id);
|
||||
} catch (\Throwable) {
|
||||
throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)");
|
||||
}
|
||||
if (microtime(true) >= $deadline) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return ['total' => count($ids), 'items' => $items];
|
||||
}
|
||||
|
||||
private function trashInfo(string $snapshotId): array {
|
||||
return json_decode($this->getObject('trash/info/' . $snapshotId), true, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
@@ -679,7 +1165,10 @@ final class Repository {
|
||||
for ($attempt = 1; $attempt <= 3; $attempt++) {
|
||||
$before = self::fileVersion($path);
|
||||
$blobs = [];
|
||||
$fh = fopen($path, 'rb');
|
||||
$fh = self::openForReading($path);
|
||||
if ($fh === false) {
|
||||
throw new RepositoryException('Cannot read ' . $path);
|
||||
}
|
||||
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
|
||||
$blobs[] = $this->storeBlob($data, $packs, $stats);
|
||||
}
|
||||
@@ -771,16 +1260,108 @@ final class Repository {
|
||||
$ranges[] = [$curPath, $curOffset, $curLength];
|
||||
$parts[] = $curParts;
|
||||
}
|
||||
foreach ($this->backend->getRanges($ranges) as $i => $data) {
|
||||
foreach ($parts[$i] as [$id, $at, $length]) {
|
||||
yield $this->decodeBlob($id, substr($data, $at, $length));
|
||||
$done = 0;
|
||||
try {
|
||||
foreach ($this->backend->getRanges($ranges) as $i => $data) {
|
||||
foreach ($parts[$i] as [$id, $at, $length]) {
|
||||
try {
|
||||
$blob = $this->decodeBlob($id, substr($data, $at, $length));
|
||||
} catch (\Exception) {
|
||||
// a read that went wrong on the way (see loadBlob()): this blob on its own
|
||||
$blob = $this->loadBlob($id);
|
||||
// its pack stream is closed before the next range is read on the same connection
|
||||
$this->packReader->close();
|
||||
}
|
||||
yield $blob;
|
||||
$done++;
|
||||
}
|
||||
}
|
||||
} catch (BackendException) {
|
||||
// A pack moved, or the read failed for a while (see loadBlob()): the rest one blob at a
|
||||
// time, each read once more after a failure.
|
||||
foreach (array_slice($ids, $done) as $id) {
|
||||
yield $this->loadBlob($id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A blob for restore(): read ahead when it could be, otherwise (or when that copy is bad) the usual way. */
|
||||
private function restoreBlob(string $id, ?ReadAheadPlan $plan): string {
|
||||
if ($plan !== null) {
|
||||
[$pack, $offset, $length] = $this->index->get($id);
|
||||
$encrypted = $plan->read('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||
if ($encrypted !== null) {
|
||||
try {
|
||||
return $this->decodeBlob($id, $encrypted);
|
||||
} catch (\Exception) {
|
||||
// read again from the location below, which reports what is wrong
|
||||
}
|
||||
}
|
||||
}
|
||||
return $this->loadBlob($id);
|
||||
}
|
||||
|
||||
private function loadBlob(string $id): string {
|
||||
[$pack, $offset, $length] = $this->index->get($id);
|
||||
return $this->decodeBlob($id, $this->packReader->read($pack, $offset, $length));
|
||||
try {
|
||||
$encrypted = $this->packReader->read($pack, $offset, $length);
|
||||
} catch (BackendException $e) {
|
||||
// Another server's cleanup may have repacked this blob into a new pack since the index
|
||||
// was loaded (a disaster recovery test while the original server still runs): add the
|
||||
// new index files, and read the blob where it is now. Or the read failed for a while
|
||||
// (a connection that died; the backend has replaced it by now): then read it once
|
||||
// more where it was. A second failure is the real one.
|
||||
try {
|
||||
$this->index->refresh();
|
||||
} catch (\Throwable) {
|
||||
// the location could not be listed either: read again where the blob was
|
||||
}
|
||||
[$moved, $offset, $length] = $this->index->get($id);
|
||||
try {
|
||||
$encrypted = $this->packReader->read($moved, $offset, $length);
|
||||
} catch (BackendException) {
|
||||
throw $e;
|
||||
}
|
||||
$pack = $moved;
|
||||
}
|
||||
try {
|
||||
return $this->decodeBlob($id, $encrypted);
|
||||
} catch (\Exception $damaged) {
|
||||
// Bytes that do not decrypt: damaged on the location, or a read that went wrong on the
|
||||
// way (a stream that lost its place: an SMB restore stopped after 17 hours like this).
|
||||
// Read exactly this blob once more, on a request of its own; only a second failure
|
||||
// means the data itself is damaged.
|
||||
$this->packReader->close();
|
||||
try {
|
||||
$again = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||
} catch (\Exception) {
|
||||
throw $damaged;
|
||||
}
|
||||
$data = $this->decodeBlob($id, $again);
|
||||
if (function_exists('OCP\Log\logger')) {
|
||||
\OCP\Log\logger('ng_backup')->warning('NG Backup: blob ' . $id . ' did not decrypt as first read from pack ' . $pack . ' at ' . $offset . '; read again, it is fine. First read: ' . self::howItDiffered($encrypted, $again));
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What a read that went wrong looked like next to the right bytes, for the log: where it starts
|
||||
* to differ, and whether the rest is zeros or the right bytes from another place (a stream that
|
||||
* lost its place).
|
||||
*/
|
||||
private static function howItDiffered(string $wrong, string $right): string {
|
||||
$at = strspn($wrong ^ $right, "\0");
|
||||
$text = strlen($wrong) . ' bytes, differ from byte ' . $at . ' of ' . strlen($right);
|
||||
$tail = substr($wrong, $at, 64);
|
||||
if ($tail === '') {
|
||||
return $text . ' (too short)';
|
||||
}
|
||||
if (trim($tail, "\0") === '') {
|
||||
return $text . ' (zeros)';
|
||||
}
|
||||
$found = strpos($right, $tail);
|
||||
return $text . ($found !== false ? ' (the right bytes from ' . ($found - $at) . ' bytes further on)' : ' (other bytes)');
|
||||
}
|
||||
|
||||
/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */
|
||||
@@ -1005,7 +1586,10 @@ final class Repository {
|
||||
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
||||
fclose($fh);
|
||||
rewind($plain);
|
||||
fgets($plain); // meta line
|
||||
$meta = json_decode((string)fgets($plain), true);
|
||||
if (is_array($meta)) {
|
||||
$this->lastMeta = [$snapshotId => $meta]; // a snapshot never changes
|
||||
}
|
||||
while (($line = fgets($plain)) !== false) {
|
||||
$entry = json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!is_array($entry) || !is_string($entry['p'] ?? null) || !self::isSafePath($entry['p'])
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Repository;
|
||||
|
||||
/**
|
||||
* A restore cannot write to where it restores to (a full disk, permissions, something in the way).
|
||||
* The backup is fine, so this never counts as damaged data: the restore stops, also with --salvage.
|
||||
*/
|
||||
final class RestoreTargetException extends \RuntimeException {
|
||||
}
|
||||
@@ -61,6 +61,16 @@ final class AlertService {
|
||||
'examples' => mb_strimwidth(implode(', ', array_merge($examples['unreadable'], $examples['changing'])), 0, 300, '…')]);
|
||||
}
|
||||
|
||||
/**
|
||||
* A backup that finished but left out files or folders because of their name.
|
||||
*
|
||||
* @param list<string> $examples
|
||||
*/
|
||||
public function runBadNames(Run $run, string $targetName, int $count, array $examples): void {
|
||||
$this->notifyAdmins('run_bad_names', 'run-' . $run->getId(), ['target' => $targetName, 'count' => (string)$count,
|
||||
'examples' => mb_strimwidth(implode(', ', $examples), 0, 300, '…')]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remember how much is stored on a location (for the settings page) and warn administrators
|
||||
* once a day when it is at 80% of its limit or more.
|
||||
@@ -91,9 +101,36 @@ final class AlertService {
|
||||
return $v === '' ? null : (int)$v;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether it is time to remind administrators of a location without a schedule: once a week
|
||||
* while another location is still backed up, every day when none is. An hour of margin, so
|
||||
* that the moment of the cron run does not push a reminder to the next day.
|
||||
*/
|
||||
public static function offReminderDue(int $last, int $now, bool $othersActive): bool {
|
||||
return $now - $last >= ($othersActive ? 7 * 86400 : 86400) - 3600;
|
||||
}
|
||||
|
||||
public function checkOff(Target $target, bool $othersActive): void {
|
||||
if ((string)$target->getSchedule() !== '') {
|
||||
return;
|
||||
}
|
||||
$key = 'off_notified_' . $target->getId();
|
||||
$now = time();
|
||||
$last = max((int)$target->getScheduleSince(), $this->appConfig->getValueInt(Application::APP_ID, $key, 0));
|
||||
if (!self::offReminderDue($last, $now, $othersActive)) {
|
||||
return;
|
||||
}
|
||||
$this->appConfig->setValueInt(Application::APP_ID, $key, $now);
|
||||
$this->notifyAdmins('schedule_off', 'off-' . $target->getId() . '-' . date('Y-m-d', $now),
|
||||
['target' => $target->getName(), 'others' => $othersActive ? '1' : '0']);
|
||||
}
|
||||
|
||||
/** At most one notification per location per day. */
|
||||
public function checkStale(Target $target): void {
|
||||
$last = $this->runs->lastFinished($target->getId(), BackupService::KIND_FULL);
|
||||
if ((string)$target->getSchedule() === '') {
|
||||
return; // no schedule is a choice, as for a location that is only connected now and then
|
||||
}
|
||||
$last =$this->runs->lastFinished($target->getId(), BackupService::KIND_FULL);
|
||||
$lastTime = $last?->getFinishedAt() ?? $target->getCreatedAt();
|
||||
if (time() - $lastTime < self::STALE_AFTER || $this->runs->findRunning($target->getId(), BackupService::KIND_FULL) !== []) {
|
||||
return;
|
||||
|
||||
@@ -34,6 +34,8 @@ use Psr\Log\LoggerInterface;
|
||||
*/
|
||||
final class BackupService {
|
||||
public const KIND_FULL = 'full';
|
||||
/** Characters of a snapshot label (the column holds 255) */
|
||||
public const LABEL_MAX = 100;
|
||||
|
||||
/** @var \WeakMap<Repository, string> the location settings each kept repository was opened with */
|
||||
private \WeakMap $openedWith;
|
||||
@@ -56,7 +58,23 @@ final class BackupService {
|
||||
$this->openedWith = $opened;
|
||||
}
|
||||
|
||||
/**
|
||||
* The running backups in the order they get their turn: first come, first served. The first one
|
||||
* runs, the others are queued; only one at a time reads the disks.
|
||||
*
|
||||
* @return list<Run>
|
||||
*/
|
||||
public function runsInTurn(): array {
|
||||
$running = $this->runs->findRunning(null, self::KIND_FULL);
|
||||
usort($running, static fn (Run $a, Run $b): int => [$a->getStartedAt(), $a->getId()] <=> [$b->getStartedAt(), $b->getId()]);
|
||||
return $running;
|
||||
}
|
||||
|
||||
public function start(Target $target, string $label = ''): Run {
|
||||
$label = trim($label);
|
||||
if (mb_strlen($label) > self::LABEL_MAX) {
|
||||
throw new \InvalidArgumentException('A label has at most ' . self::LABEL_MAX . ' characters');
|
||||
}
|
||||
if (!$this->keys->everConfirmed()) {
|
||||
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
|
||||
}
|
||||
@@ -73,7 +91,7 @@ final class BackupService {
|
||||
try {
|
||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $this->createRun($target, $label));
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('A cleanup or another start for ' . $target->getName() . ' is running; try again later');
|
||||
throw new \RuntimeException('A cleanup, a restore or another start for ' . $target->getName() . ' is running, or one was stopped by force (that clears by itself within two hours); try again later');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -204,6 +222,13 @@ final class BackupService {
|
||||
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
||||
|
||||
if ($run->getPhase() === 'db') {
|
||||
if (!isset($state['dump'])) {
|
||||
// It may have been queued for hours: the restore point has the time of its data.
|
||||
$state['files']['started'] = gmdate('c');
|
||||
// Locations from before 1.1.0 get their sealed settings at their next backup.
|
||||
$this->targets->recordSettings($repo);
|
||||
$this->targets->catchUpWrappedKey($target, $repo);
|
||||
}
|
||||
$dumper = new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_'));
|
||||
$dbStats = $state['dbStats'] ?? [];
|
||||
// With system cron the whole dump usually fits in one step (one transaction, fully
|
||||
@@ -247,8 +272,13 @@ final class BackupService {
|
||||
try {
|
||||
$this->alerts->storedBytes($target, $repo->storedBytes());
|
||||
$stats = $state['files']['stats'];
|
||||
$warnings = BackupRun::warnings($state['files']);
|
||||
if (($stats['badNames'] ?? 0) > 0) {
|
||||
$this->logger->warning(sprintf('NG Backup run %d to %s: %d left out because their name is not supported (rename them): %s',
|
||||
$run->getId(), $target->getName(), $stats['badNames'], implode(', ', $warnings['badNames'])));
|
||||
$this->alerts->runBadNames($run, $target->getName(), (int)$stats['badNames'], $warnings['badNames']);
|
||||
}
|
||||
if (($stats['unreadable'] ?? 0) > 0 || ($stats['changing'] ?? 0) > 0) {
|
||||
$warnings = BackupRun::warnings($state['files']);
|
||||
$this->logger->warning(sprintf('NG Backup run %d to %s: %d unreadable (missing from the snapshot): %s; %d changed while being read: %s',
|
||||
$run->getId(), $target->getName(), $stats['unreadable'] ?? 0, implode(', ', $warnings['unreadable']),
|
||||
$stats['changing'] ?? 0, implode(', ', $warnings['changing'])));
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\Db\Run;
|
||||
use OCP\Exceptions\AppConfigTypeConflictException;
|
||||
use OCP\IAppConfig;
|
||||
use OCP\IURLGenerator;
|
||||
|
||||
/**
|
||||
* Whether Nextcloud's background jobs run on a system cron, which backups need: a backup moves
|
||||
* forward only when a background job run takes its next step (#13). Without system cron, AJAX
|
||||
* cron runs a job only now and then while someone has Nextcloud open, and webcron only as often as
|
||||
* the outside service calls it.
|
||||
*/
|
||||
final class CronHealth {
|
||||
/** A system cron that has not run for this long has stopped (it runs every 5 minutes). */
|
||||
public const STOPPED_AFTER = 3600;
|
||||
|
||||
/**
|
||||
* A running backup that took no step for this long is waiting for background jobs. A step with
|
||||
* system cron takes at most a few minutes (BackupJob) and the job runs every minute.
|
||||
*/
|
||||
public const WAITING_AFTER = 900;
|
||||
|
||||
public function __construct(
|
||||
private IAppConfig $appConfig,
|
||||
private IURLGenerator $urls,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* mode: ajax, webcron or cron; lastRun: the last background job run (0: never); ok: a system
|
||||
* cron that ran recently; stopped: system cron is set but has not run for STOPPED_AFTER.
|
||||
*
|
||||
* @return array{mode: string, lastRun: int, ok: bool, stopped: bool, docs: string}
|
||||
*/
|
||||
public function check(?int $now = null): array {
|
||||
$now ??= time();
|
||||
$mode = $this->coreString('backgroundjobs_mode', 'ajax');
|
||||
$lastRun = (int)$this->coreString('lastcron', '0');
|
||||
$stopped = $mode === 'cron' && $now - $lastRun > self::STOPPED_AFTER;
|
||||
return [
|
||||
'mode' => $mode,
|
||||
'lastRun' => $lastRun,
|
||||
'ok' => $mode === 'cron' && !$stopped,
|
||||
'stopped' => $stopped,
|
||||
'docs' => $this->urls->linkToDocs('admin-background-jobs'),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Seconds since a running backup took its last step, when that is long enough to say it waits
|
||||
* for background jobs; null while it moves (or when it is not running).
|
||||
*/
|
||||
public static function waitingFor(Run $run, ?int $now = null): ?int {
|
||||
$idle = ($now ?? time()) - $run->getUpdatedAt();
|
||||
return $run->getStatus() === Run::RUNNING && $idle >= self::WAITING_AFTER ? $idle : null;
|
||||
}
|
||||
|
||||
/** One line for occ and the log; null when system cron runs. */
|
||||
public static function describe(array $health, ?int $now = null): ?string {
|
||||
if ($health['ok']) {
|
||||
return null;
|
||||
}
|
||||
$last = $health['lastRun'] > 0 ? 'last ran ' . self::ago(($now ?? time()) - $health['lastRun']) : 'never ran';
|
||||
$what = $health['stopped']
|
||||
? 'Background jobs are set to system cron, but cron has not run for over an hour (' . $last . ').'
|
||||
: 'Background jobs do not run on a system cron: they are set to "' . ($health['mode'] === 'webcron' ? 'Webcron' : 'AJAX') . '" and ' . $last . '.';
|
||||
return $what . ' Backups, also the scheduled ones, need a system cron that runs every 5 minutes: ' . $health['docs'];
|
||||
}
|
||||
|
||||
/** "12 min ago" style, for durations from a minute to days */
|
||||
public static function ago(int $seconds): string {
|
||||
if ($seconds < 3600) {
|
||||
return max(1, intdiv($seconds, 60)) . ' min ago';
|
||||
}
|
||||
if ($seconds < 172800) {
|
||||
return intdiv($seconds, 3600) . ' h ago';
|
||||
}
|
||||
return intdiv($seconds, 86400) . ' days ago';
|
||||
}
|
||||
|
||||
/** Core stores these with different value types across versions: read whatever is there. */
|
||||
private function coreString(string $key, string $default): string {
|
||||
try {
|
||||
return $this->appConfig->getValueString('core', $key, $default);
|
||||
} catch (AppConfigTypeConflictException) {
|
||||
return (string)$this->appConfig->getValueInt('core', $key, (int)$default);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
/** A step after the database was committed failed: the database is no longer the fresh one. */
|
||||
final class DatabaseAlreadyRestoredException extends \RuntimeException {
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\AppInfo\Application;
|
||||
use OCP\DB\QueryBuilder\IQueryBuilder;
|
||||
use OCP\IAppConfig;
|
||||
use OCP\IDBConnection;
|
||||
use Psr\Log\LoggerInterface;
|
||||
|
||||
/**
|
||||
* Keeps heavy background jobs of other apps from running while a backup reads the same disks, by
|
||||
* reserving them in Nextcloud's job list: no app is turned off, and a job that is running goes on.
|
||||
* Nextcloud itself gives up on a reservation after 12 hours, so a pause that is never ended, because
|
||||
* cron or this app is gone, ends by itself. It lasts at most MAX_SECONDS, then the jobs get REST_SECONDS.
|
||||
*/
|
||||
final class HeavyJobPause {
|
||||
public const ENABLED_KEY = 'pause_heavy_jobs';
|
||||
private const SINCE_KEY = 'heavy_jobs_paused_since';
|
||||
private const REST_KEY = 'heavy_jobs_rest_until';
|
||||
public const MAX_SECONDS = 6 * 3600;
|
||||
public const REST_SECONDS = 3600;
|
||||
/** Nextcloud runs a job again this long after it was reserved. */
|
||||
private const RESERVATION_SECONDS = 12 * 3600;
|
||||
|
||||
/** Class, or the start of a class name ending in a backslash => what to call it */
|
||||
private const JOBS = [
|
||||
'OCA\\Memories\\' => 'Memories',
|
||||
'OCA\\Files_Antivirus\\BackgroundJob\\BackgroundScanner' => 'Antivirus background scan',
|
||||
'OCA\\Recognize\\' => 'Recognize',
|
||||
'OCA\\PreviewGenerator\\' => 'Preview Generator',
|
||||
'OCA\\FullTextSearch\\' => 'Full text search',
|
||||
];
|
||||
|
||||
public const NOTHING = 'nothing';
|
||||
public const PAUSE = 'pause';
|
||||
public const RELEASE = 'release';
|
||||
public const REST = 'rest';
|
||||
|
||||
public function __construct(
|
||||
private IDBConnection $db,
|
||||
private IAppConfig $appConfig,
|
||||
private LoggerInterface $logger,
|
||||
) {
|
||||
}
|
||||
|
||||
/** @return self::NOTHING|self::PAUSE|self::RELEASE|self::REST */
|
||||
public static function decide(bool $enabled, bool $backupRunning, int $since, int $restUntil, int $now): string {
|
||||
if ($since > 0 && $now - $since >= self::MAX_SECONDS) {
|
||||
return self::REST;
|
||||
}
|
||||
if ($enabled && $backupRunning && $now >= $restUntil) {
|
||||
return self::PAUSE;
|
||||
}
|
||||
return $since > 0 ? self::RELEASE : self::NOTHING;
|
||||
}
|
||||
|
||||
/** Call regularly: pauses or ends the pause as needed. A problem here never stops a backup. */
|
||||
public function sync(bool $backupRunning): void {
|
||||
try {
|
||||
$this->apply($backupRunning, time());
|
||||
} catch (\Throwable $e) {
|
||||
$this->logger->warning('NG Backup: pausing heavy background jobs failed: ' . $e->getMessage(), ['exception' => $e]);
|
||||
}
|
||||
}
|
||||
|
||||
private function apply(bool $backupRunning, int $now): void {
|
||||
$since = $this->appConfig->getValueInt(Application::APP_ID, self::SINCE_KEY, 0);
|
||||
$restUntil = $this->appConfig->getValueInt(Application::APP_ID, self::REST_KEY, 0);
|
||||
switch (self::decide($this->enabled(), $backupRunning, $since, $restUntil, $now)) {
|
||||
case self::PAUSE:
|
||||
if ($since === 0) {
|
||||
$since = $now;
|
||||
$this->appConfig->setValueInt(Application::APP_ID, self::SINCE_KEY, $since);
|
||||
$this->logger->info('NG Backup: heavy background jobs paused while a backup runs');
|
||||
}
|
||||
$this->reserve($since, $now); // also the jobs that were added or running until now
|
||||
break;
|
||||
case self::REST:
|
||||
$this->appConfig->setValueInt(Application::APP_ID, self::REST_KEY, $now + self::REST_SECONDS);
|
||||
$this->release($since);
|
||||
$this->logger->info('NG Backup: heavy background jobs get their turn for an hour during a long backup');
|
||||
break;
|
||||
case self::RELEASE:
|
||||
$this->release($since);
|
||||
$this->logger->info('NG Backup: heavy background jobs run again');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private function reserve(int $stamp, int $now): void {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->update('jobs')
|
||||
->set('reserved_at', $qb->createNamedParameter($stamp, IQueryBuilder::PARAM_INT))
|
||||
->where($qb->expr()->lte('reserved_at', $qb->createNamedParameter($now - self::RESERVATION_SECONDS, IQueryBuilder::PARAM_INT)))
|
||||
->andWhere($this->isHeavy($qb));
|
||||
$qb->executeStatement();
|
||||
}
|
||||
|
||||
/** Only our own reservations: a job that Nextcloud reserved at another moment is running. */
|
||||
private function release(int $stamp): void {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->update('jobs')
|
||||
->set('reserved_at', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT))
|
||||
->where($qb->expr()->eq('reserved_at', $qb->createNamedParameter($stamp, IQueryBuilder::PARAM_INT)))
|
||||
->andWhere($this->isHeavy($qb));
|
||||
$qb->executeStatement();
|
||||
$this->appConfig->deleteKey(Application::APP_ID, self::SINCE_KEY);
|
||||
}
|
||||
|
||||
/** @return \OCP\DB\QueryBuilder\ICompositeExpression */
|
||||
private function isHeavy(IQueryBuilder $qb) {
|
||||
$conditions = [];
|
||||
foreach (array_keys(self::JOBS) as $class) {
|
||||
$conditions[] = str_ends_with($class, '\\')
|
||||
? $qb->expr()->like('class', $qb->createNamedParameter($this->db->escapeLikeParameter($class) . '%'))
|
||||
: $qb->expr()->eq('class', $qb->createNamedParameter($class));
|
||||
}
|
||||
return $qb->expr()->orX(...$conditions);
|
||||
}
|
||||
|
||||
public function enabled(): bool {
|
||||
return $this->appConfig->getValueString(Application::APP_ID, self::ENABLED_KEY, '1') !== '0';
|
||||
}
|
||||
|
||||
public function setEnabled(bool $enabled): void {
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::ENABLED_KEY, $enabled ? '1' : '0');
|
||||
}
|
||||
|
||||
/** @return array{enabled: bool, since: int, jobs: list<string>} what is paused now */
|
||||
public function status(): array {
|
||||
$since = $this->appConfig->getValueInt(Application::APP_ID, self::SINCE_KEY, 0);
|
||||
$jobs = [];
|
||||
if ($since > 0) {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->selectDistinct('class')->from('jobs')
|
||||
->where($qb->expr()->eq('reserved_at', $qb->createNamedParameter($since, IQueryBuilder::PARAM_INT)))
|
||||
->andWhere($this->isHeavy($qb));
|
||||
$result = $qb->executeQuery();
|
||||
while (($class = $result->fetchOne()) !== false) {
|
||||
foreach (self::JOBS as $start => $name) {
|
||||
if (str_starts_with((string)$class, $start)) {
|
||||
$jobs[$name] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
$result->closeCursor();
|
||||
}
|
||||
return ['enabled' => $this->enabled(), 'since' => $since, 'jobs' => array_map('strval', array_keys($jobs))];
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,9 @@ final class KeyService {
|
||||
private const KEY_DELETE_DELAY = 'delete_delay_days';
|
||||
private const KEY_KIT_VERSION = 'recovery_kit_version';
|
||||
private const KEY_KIT_CODE = 'recovery_kit_code';
|
||||
/** Set once a location was created or adopted with this key: from then on it is never replaced. */
|
||||
private const KEY_USED = 'master_key_used';
|
||||
private const KEY_WRAPPED_PENDING = 'master_key_wrapped_pending';
|
||||
public const DEFAULT_DELETE_DELAY = 7;
|
||||
/**
|
||||
* The location holds the passphrase-wrapped key too, so whoever can read it can try
|
||||
@@ -83,28 +86,100 @@ final class KeyService {
|
||||
* @param array<string, mixed> $kit as produced by recoveryKit()
|
||||
* @throws \OCA\NgBackup\Crypto\CryptoException wrong passphrase, or a damaged/foreign kit
|
||||
*/
|
||||
public function importRecoveryKit(array $kit, #[\SensitiveParameter] string $passphrase): void {
|
||||
if ($this->isInitialized()) {
|
||||
public function importRecoveryKit(array $kit, #[\SensitiveParameter] string $passphrase, bool $replaceUnused = false): void {
|
||||
if ($this->isInitialized() && !$replaceUnused) {
|
||||
throw new \RuntimeException('A backup key already exists');
|
||||
}
|
||||
if (!isset($kit['wrapped_key'])) {
|
||||
throw new \InvalidArgumentException('Not a NG Backup recovery kit file');
|
||||
}
|
||||
$keys = KeyRing::unwrapAny($kit['wrapped_key'], $passphrase);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
||||
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
||||
$this->cached = $keys;
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, json_encode($kit['wrapped_key'], JSON_THROW_ON_ERROR), true, true);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
||||
$this->discardUnused($keys, $replaceUnused);
|
||||
// Kits from before this field existed fall back to the default.
|
||||
$delay = (int)($kit['delete_delay_days'] ?? self::DEFAULT_DELETE_DELAY);
|
||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, ($delay >= 1 && $delay <= 365) ? $delay : self::DEFAULT_DELETE_DELAY, true);
|
||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, (int)($kit['kit_version'] ?? 1), true);
|
||||
$this->store($keys, $kit['wrapped_key'], (int)($kit['delete_delay_days'] ?? self::DEFAULT_DELETE_DELAY), (int)($kit['kit_version'] ?? 1));
|
||||
// Having and successfully using a real kit file is stronger proof than the normal
|
||||
// download-and-confirm flow it replaces.
|
||||
$this->confirmRecoveryKit('cli-import');
|
||||
}
|
||||
|
||||
/**
|
||||
* Disaster recovery without the recovery kit: adopt the master key that a passphrase opened
|
||||
* from the key slots in a location's config (TargetService::addWithPassphrase). There is no
|
||||
* kit, so none is confirmed: backups wait until a new kit is downloaded and confirmed.
|
||||
*
|
||||
* @param ?int $deleteDelayDays from the location's sealed settings, null when it has none
|
||||
*/
|
||||
public function adoptKey(KeyRing $keys, array $wrappedKey, ?int $deleteDelayDays, bool $replaceUnused = false): void {
|
||||
if ($this->isInitialized() && !$replaceUnused) {
|
||||
throw new \RuntimeException('A backup key already exists');
|
||||
}
|
||||
$this->discardUnused($keys, $replaceUnused);
|
||||
$this->store($keys, $wrappedKey, $deleteDelayDays ?? self::DEFAULT_DELETE_DELAY, 1);
|
||||
}
|
||||
|
||||
/** Whether a location was ever created or adopted with this key (TargetService::keyIsUnused() also looks at the database). */
|
||||
public function wasUsed(): bool {
|
||||
return $this->appConfig->getValueBool(Application::APP_ID, self::KEY_USED, false, true);
|
||||
}
|
||||
|
||||
public function markUsed(): void {
|
||||
if (!$this->wasUsed()) {
|
||||
$this->appConfig->setValueBool(Application::APP_ID, self::KEY_USED, true, true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Locations that missed the last passphrase change (offline at the time): they still hold
|
||||
* the previous slots until the change reaches them, which is tried again at each backup.
|
||||
*/
|
||||
public function wrappedKeyPending(int $targetId): bool {
|
||||
return $this->appConfig->getValueBool(Application::APP_ID, self::KEY_WRAPPED_PENDING . '_' . $targetId);
|
||||
}
|
||||
|
||||
/** One flag per location, so that two processes never overwrite each other's. */
|
||||
public function setWrappedKeyPending(int $targetId, bool $pending): void {
|
||||
$key = self::KEY_WRAPPED_PENDING . '_' . $targetId;
|
||||
if ($pending) {
|
||||
$this->appConfig->setValueBool(Application::APP_ID, $key, true);
|
||||
} elseif ($this->wrappedKeyPending($targetId)) {
|
||||
$this->appConfig->deleteKey(Application::APP_ID, $key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A key made by mistake on a server that is being restored: remove it, so the key of the
|
||||
* backups can take its place. Only after the new key was opened with its passphrase, and
|
||||
* only when the caller checked that the old key never protected anything
|
||||
* (TargetService::keyIsUnused()). Its passphrases and recovery kit become worthless.
|
||||
*/
|
||||
private function discardUnused(KeyRing $replacement, bool $replaceUnused): void {
|
||||
if (!$replaceUnused || !$this->isInitialized()) {
|
||||
return;
|
||||
}
|
||||
if ($this->wasUsed()) {
|
||||
throw new \RuntimeException('The backup key of this server has been used for a backup location: it is never replaced');
|
||||
}
|
||||
if ($this->fingerprintOf($replacement) === $this->fingerprint()) {
|
||||
throw new \RuntimeException('This server already has this backup key: nothing to replace');
|
||||
}
|
||||
foreach ([self::KEY_MASTER, self::KEY_WRAPPED, self::KEY_CONFIRMED, self::KEY_FINGERPRINT, self::KEY_DELETE_DELAY,
|
||||
self::KEY_KIT_VERSION, self::KEY_KIT_CODE, self::KEY_USED] as $key) {
|
||||
$this->appConfig->deleteKey(Application::APP_ID, $key);
|
||||
}
|
||||
$this->cached = null;
|
||||
}
|
||||
|
||||
private function store(KeyRing $keys, array $wrappedKey, int $deleteDelayDays, int $kitVersion): void {
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
||||
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
||||
$this->cached = $keys;
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, json_encode($wrappedKey, JSON_THROW_ON_ERROR), true, true);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY,
|
||||
($deleteDelayDays >= 1 && $deleteDelayDays <= 365) ? $deleteDelayDays : self::DEFAULT_DELETE_DELAY, true);
|
||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, max(1, $kitVersion), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* The master key and the recovery kit code are stored encrypted with Nextcloud's own
|
||||
* instance secret (ICrypto), not with themselves; the wrapped key is not (it is sealed with
|
||||
|
||||
@@ -43,8 +43,10 @@ final class KeySlotService {
|
||||
foreach ($this->targets->list() as $t) {
|
||||
try {
|
||||
$this->targets->repository($t)->updateWrappedKey($wrapped);
|
||||
$this->keys->setWrappedKeyPending($t->getId(), false);
|
||||
} catch (\Throwable $e) {
|
||||
$failed[] = $t->getName();
|
||||
$this->keys->setWrappedKeyPending($t->getId(), true);
|
||||
}
|
||||
}
|
||||
$label = '';
|
||||
@@ -55,7 +57,7 @@ final class KeySlotService {
|
||||
}
|
||||
$this->alerts->securityEvent($event, ['slot' => (string)$slot, 'label' => $label]);
|
||||
if ($failed !== []) {
|
||||
throw new \RuntimeException('Saved, but these locations could not be updated (they still accept the previous passphrases): ' . implode(', ', $failed));
|
||||
throw new \RuntimeException('Saved, but these locations could not be updated (they still accept the previous passphrases; the change is tried again at their next backup): ' . implode(', ', $failed));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,6 +76,15 @@ final class LeaseService {
|
||||
$qb->executeStatement();
|
||||
}
|
||||
|
||||
/** Remove the shared leases on $name, for a caller that knows their holders were stopped. */
|
||||
public function dropShared(string $name): void {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->delete('ngb_leases')
|
||||
->where($qb->expr()->eq('name', $qb->createNamedParameter($name)))
|
||||
->andWhere($qb->expr()->eq('mode', $qb->createNamedParameter(self::SHARED)));
|
||||
$qb->executeStatement();
|
||||
}
|
||||
|
||||
/** Wraps $refresh so calling it often (per file, per row batch) only touches the database every $seconds. */
|
||||
public static function throttled(callable $refresh, int $seconds = 60): \Closure {
|
||||
$last = time();
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
/**
|
||||
* When a location is backed up: "" (off), "HH:MM" (daily) or "HH:MM/N" (from HH:MM every N hours,
|
||||
* starting over at HH:MM the next day), in the location's time zone.
|
||||
*/
|
||||
final class LocationSchedule {
|
||||
/** Attempts per moment after a failure, an hour apart, instead of one every few minutes. */
|
||||
public const MAX_ATTEMPTS = 3;
|
||||
public const RETRY_AFTER = 3600;
|
||||
|
||||
/** @throws \InvalidArgumentException */
|
||||
public static function normalize(string $schedule): string {
|
||||
$schedule = trim($schedule);
|
||||
if ($schedule === '' || strtolower($schedule) === 'off') {
|
||||
return '';
|
||||
}
|
||||
if (preg_match('/^(([01]\d|2[0-3]):[0-5]\d)(?:\/(\d{1,2}))?$/', $schedule, $m) !== 1) {
|
||||
throw new \InvalidArgumentException('Use "off", HH:MM for once a day, or HH:MM/N for every N hours from HH:MM');
|
||||
}
|
||||
$hours = (int)($m[3] ?? 24);
|
||||
if ($hours < 1 || $hours > 24) {
|
||||
throw new \InvalidArgumentException('Every 1 to 24 hours');
|
||||
}
|
||||
return $hours === 24 ? $m[1] : $m[1] . '/' . $hours;
|
||||
}
|
||||
|
||||
/**
|
||||
* The most recent scheduled moment at or before $now; null without a schedule.
|
||||
*
|
||||
* @param non-empty-string $zone
|
||||
*/
|
||||
public static function lastMoment(string $schedule, string $zone, int $now): ?int {
|
||||
if ($schedule === '') {
|
||||
return null;
|
||||
}
|
||||
$tz = new \DateTimeZone($zone);
|
||||
$today = (new \DateTimeImmutable('@' . $now))->setTimezone($tz);
|
||||
$moments = array_merge(self::moments($schedule, $today->modify('-1 day'), $tz), self::moments($schedule, $today, $tz));
|
||||
$last = null;
|
||||
foreach ($moments as $moment) {
|
||||
if ($moment <= $now && ($last === null || $moment > $last)) {
|
||||
$last = $moment;
|
||||
}
|
||||
}
|
||||
return $last;
|
||||
}
|
||||
|
||||
/** @return list<int> the moments of one day */
|
||||
private static function moments(string $schedule, \DateTimeImmutable $day, \DateTimeZone $tz): array {
|
||||
[$time, $every] = array_pad(explode('/', $schedule), 2, '24');
|
||||
[$hour, $minute] = array_map('intval', explode(':', $time));
|
||||
$moments = [];
|
||||
for ($h = $hour; $h < 24; $h += (int)$every) {
|
||||
$moments[] = (new \DateTimeImmutable($day->format('Y-m-d') . sprintf(' %02d:%02d', $h, $minute), $tz))->getTimestamp();
|
||||
}
|
||||
return $moments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a backup has to start for $moment. One that was made since then is enough, also when
|
||||
* several moments went by during a long backup. A moment from before the schedule was set does
|
||||
* not count: choosing a time must not start a backup right away.
|
||||
*/
|
||||
public static function isDue(int $moment, int $since, ?int $lastDoneStarted, int $attempts, ?int $lastAttemptFinished, int $now): bool {
|
||||
if ($moment < $since || ($lastDoneStarted !== null && $lastDoneStarted >= $moment) || $attempts >= self::MAX_ATTEMPTS) {
|
||||
return false;
|
||||
}
|
||||
return $attempts === 0 || $now - (int)$lastAttemptFinished >= self::RETRY_AFTER;
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ use OCA\NgBackup\AppInfo\Application;
|
||||
use OCA\NgBackup\Db\RunMapper;
|
||||
use OCA\NgBackup\Db\SnapshotMapper;
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCP\IAppConfig;
|
||||
use OCP\Lock\ILockingProvider;
|
||||
use OCP\Lock\LockedException;
|
||||
@@ -33,6 +34,54 @@ final class PruneService {
|
||||
) {
|
||||
}
|
||||
|
||||
/** Characters of a note on a snapshot (the column holds 1000) */
|
||||
public const NOTE_MAX = 500;
|
||||
|
||||
/** Pin or unpin a snapshot: the retention policy leaves a pinned one alone. */
|
||||
public function setPinned(Target $target, string $snapshotId, bool $pinned): void {
|
||||
$changed = $this->changeCatalog($target, function (Repository $repo) use ($snapshotId, $pinned): bool {
|
||||
$was = in_array($snapshotId, $repo->pinnedSnapshots(), true);
|
||||
$repo->setPinned($snapshotId, $pinned);
|
||||
return $was !== $pinned;
|
||||
});
|
||||
$this->targets->syncSnapshot($target, $snapshotId, pinned: $pinned);
|
||||
if ($changed) {
|
||||
// Unpinning opens the way to removing it, so every administrator hears about that.
|
||||
$this->alerts->securityEvent($pinned ? 'snapshot_pinned' : 'snapshot_unpinned', ['target' => $target->getName(), 'snapshot' => $snapshotId], !$pinned);
|
||||
}
|
||||
}
|
||||
|
||||
public function setNote(Target $target, string $snapshotId, string $note): void {
|
||||
$note = trim((string)preg_replace('/[\x00-\x1f\x7f]+/', ' ', $note));
|
||||
if (!mb_check_encoding($note, 'UTF-8')) {
|
||||
throw new \InvalidArgumentException('A note must be valid UTF-8');
|
||||
}
|
||||
if (mb_strlen($note) > self::NOTE_MAX) {
|
||||
throw new \InvalidArgumentException('A note has at most ' . self::NOTE_MAX . ' characters');
|
||||
}
|
||||
$this->changeCatalog($target, function (Repository $repo) use ($snapshotId, $note): bool {
|
||||
$repo->setSnapshotNote($snapshotId, $note);
|
||||
return true;
|
||||
});
|
||||
$this->targets->syncSnapshot($target, $snapshotId, note: $note);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write to the catalog next to a running backup or restore, which only takes it for a moment to
|
||||
* commit. Not during a cleanup, which holds a repository opened earlier and writes from that.
|
||||
*
|
||||
* @param callable(Repository): bool $change
|
||||
*/
|
||||
private function changeCatalog(Target $target, callable $change): bool {
|
||||
try {
|
||||
return $this->leases->with(self::lockKey($target), LeaseService::SHARED, 300,
|
||||
fn (): bool => $this->leases->with(self::catalogLockKey($target), LeaseService::EXCLUSIVE, 300,
|
||||
fn (): bool => $change($this->targets->repository($target))));
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' or another change to its restore points is in progress; try again later');
|
||||
}
|
||||
}
|
||||
|
||||
public function untrash(Target $target, string $snapshotId): void {
|
||||
$lease = $this->lockExclusive($target, 300);
|
||||
try {
|
||||
@@ -87,6 +136,7 @@ final class PruneService {
|
||||
$s->setCreatedAt((int)strtotime((string)($meta['time'] ?? 'now')));
|
||||
$s->setFiles((int)($meta['stats']['files'] ?? 0));
|
||||
$s->setBytes((int)($meta['stats']['bytes'] ?? 0));
|
||||
$s->setNote($repo->snapshotNotes()[$snapshotId] ?? null);
|
||||
try {
|
||||
$this->snapshots->insert($s);
|
||||
} catch (\Throwable) {
|
||||
@@ -208,7 +258,7 @@ final class PruneService {
|
||||
$meta = $repo->snapshotMeta($id);
|
||||
$times[$id] = (int)strtotime((string)($meta['time'] ?? '@0'));
|
||||
}
|
||||
$keep = $this->policy()->keep($times);
|
||||
$keep = $this->policy()->keepWithPinned($times, $repo->pinnedSnapshots());
|
||||
$forget = array_values(array_diff(array_keys($times), array_keys($keep)));
|
||||
$actor = $scheduled ? 'retention policy' : $this->alerts->actor();
|
||||
$purged = [];
|
||||
|
||||
@@ -27,9 +27,14 @@ final class RestoreFullService {
|
||||
* Config keys that must keep pointing at THIS installation's actual, already-working
|
||||
* database and data directory (set by occ maintenance:install), never at the old server's.
|
||||
*/
|
||||
/** App setting: when a full restore started that has not completed (see restoreFull()). */
|
||||
private const INTERRUPTED_KEY = 'restore_full_started';
|
||||
|
||||
private const CONNECTION_KEYS = [
|
||||
'dbtype', 'dbhost', 'dbport', 'dbname', 'dbuser', 'dbpassword', 'dbpersistent',
|
||||
'dbtableprefix', 'dbdriveroptions', 'datadirectory',
|
||||
// set by restoreFull() itself for the length of the restore
|
||||
'maintenance',
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
@@ -43,80 +48,178 @@ final class RestoreFullService {
|
||||
) {
|
||||
}
|
||||
|
||||
/** Apps that only exist in a particular setup and are never needed elsewhere. */
|
||||
private const ENVIRONMENT_APPS = ['nextcloud-aio'];
|
||||
|
||||
/**
|
||||
* Whether this installation can receive $snapshotId: same Nextcloud major version, and every
|
||||
* app that was enabled at backup time already installed here (its tables must exist before
|
||||
* the database restore can write into them).
|
||||
* Whether this installation can receive $snapshotId, all problems at once:
|
||||
* - the same Nextcloud major version is required;
|
||||
* - apps that were enabled at backup time but are not here (they work again once installed);
|
||||
* - tables in the backup that do not exist here: a backup made from 1.0.0-rc.2 on records
|
||||
* their structure and the restore creates them, an older one cannot (missingTables).
|
||||
*
|
||||
* @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list<string>}
|
||||
* With $salvage, a database dump that cannot be read does not block: dbUnreadable says why, and
|
||||
* the restore then salvages the files instead (see restoreFull()).
|
||||
*
|
||||
* @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list<string>, missingTables:list<string>, dbUnreadable?:string}
|
||||
*/
|
||||
public function checkCompatibility(Target $target, string $snapshotId): array {
|
||||
$meta = $this->targets->repository($target)->snapshotMeta($snapshotId);
|
||||
public function checkCompatibility(Target $target, string $snapshotId, bool $skipMissingTables = false, bool $salvage = false): array {
|
||||
$repo = $this->targets->repository($target);
|
||||
$meta = $repo->snapshotMeta($snapshotId);
|
||||
$currentMajor = explode('.', $this->config->getSystemValueString('version'))[0];
|
||||
$snapshotMajor = explode('.', (string)($meta['nextcloud'] ?? ''))[0];
|
||||
$enabled = array_flip($this->appManager->getEnabledApps());
|
||||
$missingApps = array_values(array_diff(array_keys($meta['apps'] ?? []), array_keys($enabled)));
|
||||
$missingApps = array_values(array_diff(array_keys($meta['apps'] ?? []), array_keys($enabled), self::ENVIRONMENT_APPS));
|
||||
$missingTables = [];
|
||||
$prefixMismatch = null;
|
||||
$dbUnreadable = null;
|
||||
$manifest = null;
|
||||
if (isset($meta['db'])) {
|
||||
try {
|
||||
$manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
||||
} catch (\Throwable $e) {
|
||||
if (!$salvage) {
|
||||
throw $e;
|
||||
}
|
||||
$dbUnreadable = 'the database dump cannot be read: ' . $e->getMessage();
|
||||
}
|
||||
}
|
||||
if (is_array($manifest)) {
|
||||
$prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_');
|
||||
$sourcePrefix = DbDumper::dumpPrefix($manifest);
|
||||
if ($sourcePrefix !== null && $sourcePrefix !== $prefix) {
|
||||
$prefixMismatch = [$sourcePrefix, $prefix];
|
||||
}
|
||||
$dumper = new DbDumper($this->db->getInner(), $prefix);
|
||||
$missingTables = array_values(array_filter($dumper->missingTables($manifest),
|
||||
fn (string $t) => !isset($manifest['tables'][$t]['schema']) && !str_starts_with($t, $prefix . 'ngb_')));
|
||||
}
|
||||
return [
|
||||
'ok' => $currentMajor === $snapshotMajor && $missingApps === [],
|
||||
'ok' => $currentMajor === $snapshotMajor && $prefixMismatch === null && ($missingTables === [] || $skipMissingTables),
|
||||
'currentMajor' => $currentMajor,
|
||||
'snapshotMajor' => $snapshotMajor,
|
||||
'missingApps' => $missingApps,
|
||||
];
|
||||
'missingTables' => $prefixMismatch === null ? $missingTables : [],
|
||||
'prefixMismatch' => $prefixMismatch,
|
||||
] + ($dbUnreadable !== null ? ['dbUnreadable' => $dbUnreadable] : []);
|
||||
}
|
||||
|
||||
/** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool} */
|
||||
public function restoreFull(Target $target, string $snapshotId): array {
|
||||
$check = $this->checkCompatibility($target, $snapshotId);
|
||||
if (!$check['ok']) {
|
||||
throw new \RuntimeException($this->describeBlockers($check));
|
||||
/**
|
||||
* With $salvage, restore whatever can be restored instead of stopping:
|
||||
* - a file that cannot be read or decrypted is left out and listed (skipped);
|
||||
* - when the database cannot be restored here (its dump is unreadable or damaged, or it does
|
||||
* not fit this installation), the fresh database stays as it is, and the data directory and
|
||||
* config of the backup go to a folder of their own (salvageDir) instead: the files are what
|
||||
* matters most, and putting Nextcloud back together from there is work by hand;
|
||||
* - a config.php that cannot be read is not merged (configError).
|
||||
*
|
||||
* @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list<string>, maintenanceLeftOn:bool, skipped:array<string, string>, dbError:?string, salvageDir:?string, configError:?string}
|
||||
*/
|
||||
public function restoreFull(Target $target, string $snapshotId, bool $skipMissingTables = false, bool $salvage = false): array {
|
||||
// Maintenance mode for the whole restore: background jobs (cron.php stops in maintenance
|
||||
// mode), page visits (AJAX cron) and users must not write into tables or the data
|
||||
// directory while they are being replaced; a row written in between would collide with a
|
||||
// restored one or be silently lost. Switched off after a restore that completed. After an
|
||||
// error it stays on: the instance may be partly restored, and must not be used like that.
|
||||
// occ only offers app commands outside maintenance mode, so the admin switches it off
|
||||
// right before running the restore again (see RestoreFull). Already on before: left on.
|
||||
// On before the checks below, so that a run again after an error leaves the partly
|
||||
// restored instance open for seconds at most; off again when they refuse (nothing changed).
|
||||
// A marker in ng_backup's own app settings (which the restore keeps) says a restore started
|
||||
// and did not complete: then the instance may be partly restored, and a check that refuses
|
||||
// a run again must not open it.
|
||||
$appConfig = \OCP\Server::get(\OCP\IAppConfig::class);
|
||||
$interrupted = $appConfig->getValueInt(\OCA\NgBackup\AppInfo\Application::APP_ID, self::INTERRUPTED_KEY, 0) > 0;
|
||||
$wasInMaintenance = $this->config->getSystemValueBool('maintenance', false);
|
||||
$this->config->setSystemValue('maintenance', true);
|
||||
$reopen = function () use ($wasInMaintenance, $interrupted): void {
|
||||
if (!$wasInMaintenance && !$interrupted) {
|
||||
$this->config->setSystemValue('maintenance', false);
|
||||
}
|
||||
};
|
||||
try {
|
||||
$check = $this->checkCompatibility($target, $snapshotId, $skipMissingTables, $salvage);
|
||||
} catch (\Throwable $e) {
|
||||
$reopen();
|
||||
throw $e;
|
||||
}
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId) {
|
||||
if (!$check['ok']) {
|
||||
if (!$salvage) {
|
||||
$reopen();
|
||||
throw new \RuntimeException($this->describeBlockers($check));
|
||||
}
|
||||
// The database cannot go into this installation; the files can.
|
||||
$check['dbUnreadable'] = 'the database cannot be restored here: ' . $this->describeBlockers($check);
|
||||
}
|
||||
if ($interrupted) {
|
||||
// A restore that was killed leaves its leases, for up to two hours: they would keep
|
||||
// the first backup of the recovered server waiting after this restore completes.
|
||||
// In maintenance mode, on a partly restored instance, nothing else reads the location.
|
||||
$this->leases->dropShared(PruneService::lockKey($target));
|
||||
}
|
||||
$appConfig->setValueInt(\OCA\NgBackup\AppInfo\Application::APP_ID, self::INTERRUPTED_KEY, time());
|
||||
$result = $this->restoreInMaintenance($target, $snapshotId, $skipMissingTables, $check, $salvage);
|
||||
$appConfig->deleteKey(\OCA\NgBackup\AppInfo\Application::APP_ID, self::INTERRUPTED_KEY);
|
||||
if (!$wasInMaintenance) {
|
||||
$this->config->setSystemValue('maintenance', false);
|
||||
}
|
||||
return $result + ['maintenanceLeftOn' => $wasInMaintenance];
|
||||
}
|
||||
|
||||
/** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool, missingApps:list<string>, skipped:array<string, string>, dbError:?string, salvageDir:?string, configError:?string} */
|
||||
private function restoreInMaintenance(Target $target, string $snapshotId, bool $skipMissingTables, array $check, bool $salvage): array {
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $skipMissingTables, $check, $salvage) {
|
||||
$beat = LeaseService::throttled($refresh);
|
||||
$repo = $this->targets->repository($target);
|
||||
$meta = $repo->snapshotMeta($snapshotId);
|
||||
|
||||
$dbReport = null;
|
||||
$manifest = [];
|
||||
if (isset($meta['db'])) {
|
||||
$manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
||||
$prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_');
|
||||
// ng_backup's own tables (and its own rows in the shared appconfig table) must
|
||||
// survive this restore untouched: they describe the backup/key/target that is
|
||||
// actively performing it right now, on THIS installation, not the one that was
|
||||
// backed up. Overwriting them mid-restore would pull the rug from under it.
|
||||
foreach (array_keys($manifest['tables']) as $table) {
|
||||
/** @var string $table */
|
||||
if (str_starts_with($table, $prefix . 'ngb_')) {
|
||||
unset($manifest['tables'][$table]);
|
||||
}
|
||||
}
|
||||
// Sequences are keyed separately from tables; excluding a table does not exclude
|
||||
// its sequence, which would otherwise rewind a retained ngb_* table's sequence
|
||||
// below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()).
|
||||
foreach (array_keys($manifest['sequences'] ?? []) as $seq) {
|
||||
/** @var string $seq */
|
||||
if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) {
|
||||
unset($manifest['sequences'][$seq]);
|
||||
}
|
||||
}
|
||||
$ownAppConfig = $this->preserveOwnAppConfig();
|
||||
$dumper = new DbDumper($this->db->getInner(), $prefix);
|
||||
$dbError = $check['dbUnreadable'] ?? null;
|
||||
$skipped = $salvage ? [] : null;
|
||||
$dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/');
|
||||
if ($dbError === null) {
|
||||
try {
|
||||
$dumper->restore($repo, $manifest, $dbReport, $beat);
|
||||
} finally {
|
||||
$this->restoreOwnAppConfig($ownAppConfig);
|
||||
$this->restoreDatabase($repo, $meta, $manifest, $dbReport, $beat, $skipMissingTables);
|
||||
} catch (DatabaseAlreadyRestoredException $e) {
|
||||
throw $e;
|
||||
} catch (\Throwable $e) {
|
||||
if (!$salvage) {
|
||||
throw $e;
|
||||
}
|
||||
// DbDumper::restore() rolled its transaction back: the database is the fresh one.
|
||||
$dbError = 'the database could not be restored: ' . $e->getMessage();
|
||||
$dbReport = null;
|
||||
}
|
||||
}
|
||||
if ($dbError !== null) {
|
||||
$salvageDir = $dataDir . '/ngb-salvage-' . substr($snapshotId, 0, 12);
|
||||
$dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $salvageDir . '/data', stripPrefix: true, heartbeat: $beat, skipped: $skipped, keepUnchanged: true);
|
||||
$this->restore->restoreToDirectory($target, $snapshotId, 'config', $salvageDir . '/config', stripPrefix: true, heartbeat: $beat, skipped: $skipped);
|
||||
return ['dbReport' => ['skipped' => [], 'restored' => [], 'created' => [], 'missing' => []], 'dataFiles' => $dataFiles, 'configKeys' => 0,
|
||||
'dbConsistent' => true, 'missingApps' => $check['missingApps'], 'skipped' => $skipped ?? [], 'dbError' => $dbError,
|
||||
'salvageDir' => $salvageDir, 'configError' => null];
|
||||
}
|
||||
|
||||
// Read (but do not yet apply) the backed-up config.php before touching the data
|
||||
// directory: once applied, 'secret' may differ from what ng_backup's own Target
|
||||
// options were encrypted with (see restoreFull()'s docblock), and every further
|
||||
// repository access through $this->targets would fail to decrypt them.
|
||||
/** @var array<string, mixed> $backupConfig */
|
||||
$backupConfig = $this->fetchBackupConfig($target, $snapshotId);
|
||||
$configError = null;
|
||||
try {
|
||||
/** @var array<string, mixed> $backupConfig */
|
||||
$backupConfig = $this->fetchBackupConfig($target, $snapshotId, $salvage);
|
||||
} catch (\OCA\NgBackup\Repository\RestoreTargetException $e) {
|
||||
throw $e; // nowhere to put it: not a config.php that cannot be read
|
||||
} catch (\Throwable $e) {
|
||||
if (!$salvage) {
|
||||
throw $e;
|
||||
}
|
||||
$configError = $e->getMessage();
|
||||
$backupConfig = [];
|
||||
}
|
||||
|
||||
$dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/');
|
||||
$dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat);
|
||||
$dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat, skipped: $skipped, keepUnchanged: true);
|
||||
|
||||
// Last step: nothing after this may need $this->targets->repository($target) again.
|
||||
if ($backupConfig !== []) {
|
||||
@@ -125,16 +228,76 @@ final class RestoreFullService {
|
||||
);
|
||||
}
|
||||
|
||||
return ['dbReport' => $dbReport ?? ['skipped' => [], 'restored' => []], 'dataFiles' => $dataFiles, 'configKeys' => count($backupConfig),
|
||||
'dbConsistent' => (bool)($manifest['consistent'] ?? true)];
|
||||
return ['dbReport' => $dbReport ?? ['skipped' => [], 'restored' => [], 'created' => [], 'missing' => []], 'dataFiles' => $dataFiles, 'configKeys' => count($backupConfig),
|
||||
'dbConsistent' => (bool)($manifest['consistent'] ?? true), 'missingApps' => $check['missingApps'],
|
||||
'skipped' => $skipped ?? [], 'dbError' => null, 'salvageDir' => null, 'configError' => $configError];
|
||||
});
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> config.php's $CONFIG from the backup, minus the connection keys */
|
||||
private function fetchBackupConfig(Target $target, string $snapshotId): array {
|
||||
/**
|
||||
* Replace the database with the snapshot's dump (one transaction, see DbDumper::restore()).
|
||||
*
|
||||
* @param array<string, mixed> $meta the snapshot's meta record
|
||||
* @param array $manifest set to the dump's manifest
|
||||
*/
|
||||
private function restoreDatabase(\OCA\NgBackup\Repository\Repository $repo, array $meta, array &$manifest, ?array &$dbReport, callable $beat, bool $skipMissingTables): void {
|
||||
if (!isset($meta['db'])) {
|
||||
return;
|
||||
}
|
||||
$manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
||||
$prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_');
|
||||
// ng_backup's own tables (and its own rows in the shared appconfig table) must
|
||||
// survive this restore untouched: they describe the backup/key/target that is
|
||||
// actively performing it right now, on THIS installation, not the one that was
|
||||
// backed up. Overwriting them mid-restore would pull the rug from under it.
|
||||
foreach (array_keys($manifest['tables']) as $table) {
|
||||
/** @var string $table */
|
||||
if (str_starts_with($table, $prefix . 'ngb_')) {
|
||||
unset($manifest['tables'][$table]);
|
||||
}
|
||||
}
|
||||
// Sequences are keyed separately from tables; excluding a table does not exclude
|
||||
// its sequence, which would otherwise rewind a retained ngb_* table's sequence
|
||||
// below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()).
|
||||
foreach (array_keys($manifest['sequences'] ?? []) as $seq) {
|
||||
/** @var string $seq */
|
||||
if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) {
|
||||
unset($manifest['sequences'][$seq]);
|
||||
}
|
||||
}
|
||||
$ownAppConfig = $this->preserveOwnAppConfig();
|
||||
$dumper = new DbDumper($this->db->getInner(), $prefix);
|
||||
try {
|
||||
$dumper->restore($repo, $manifest, $dbReport, $beat, $skipMissingTables);
|
||||
} catch (\Throwable $e) {
|
||||
try {
|
||||
$this->restoreOwnAppConfig($ownAppConfig);
|
||||
} catch (\Throwable) {
|
||||
// rolled back, so they are still there: the error that brought us here is the one to report
|
||||
}
|
||||
throw $e;
|
||||
}
|
||||
try {
|
||||
$this->restoreOwnAppConfig($ownAppConfig);
|
||||
} catch (\Throwable $e) {
|
||||
// The old server's database is in place by now: not a case for "the database stays as
|
||||
// the fresh installation made it" under --salvage. A run again restores these rows.
|
||||
throw new DatabaseAlreadyRestoredException('The database was restored, but NG Backup could not put back its own settings: ' . $e->getMessage(), 0, $e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, mixed> config.php's $CONFIG from the backup, minus the connection keys
|
||||
* @throws \RuntimeException with $salvage, when config.php itself cannot be read
|
||||
*/
|
||||
private function fetchBackupConfig(Target $target, string $snapshotId, bool $salvage = false): array {
|
||||
$tmp = sys_get_temp_dir() . '/ngb-restore-config-' . bin2hex(random_bytes(6));
|
||||
try {
|
||||
$this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true);
|
||||
$skipped = $salvage ? [] : null;
|
||||
$this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true, skipped: $skipped);
|
||||
if (isset($skipped['config/config.php'])) {
|
||||
throw new \RuntimeException('config.php cannot be read: ' . $skipped['config/config.php']);
|
||||
}
|
||||
$path = $tmp . '/config.php';
|
||||
if (!is_file($path)) {
|
||||
return []; // nothing to merge; the live config.php stays as occ maintenance:install made it
|
||||
@@ -209,11 +372,15 @@ final class RestoreFullService {
|
||||
if ($check['currentMajor'] !== $check['snapshotMajor']) {
|
||||
$parts[] = "this installation is Nextcloud {$check['currentMajor']}, the backup is from {$check['snapshotMajor']} (same major version required)";
|
||||
}
|
||||
if ($check['missingApps'] !== []) {
|
||||
$parts[] = 'install these apps first (their tables must exist before the database restore can use them): '
|
||||
. implode(', ', array_map(fn (string $a) => "occ app:install $a", $check['missingApps']));
|
||||
if (($check['prefixMismatch'] ?? null) !== null) {
|
||||
[$source, $here] = $check['prefixMismatch'];
|
||||
$parts[] = "the backup's tables use the prefix '$source', this installation '$here': install Nextcloud again with 'dbtableprefix' => '$source' in config/config.php before occ maintenance:install, and restore again";
|
||||
}
|
||||
return implode('; ', $parts);
|
||||
if ($check['missingTables'] !== []) {
|
||||
$prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_');
|
||||
$parts[] = DbDumper::describeMissing($check['missingTables'], $prefix);
|
||||
}
|
||||
return implode("\n", $parts);
|
||||
}
|
||||
|
||||
private function withSharedLock(Target $target, callable $fn): mixed {
|
||||
|
||||
@@ -33,6 +33,9 @@ final class RestoreService {
|
||||
public const MODE_MERGE = 'merge';
|
||||
public const MODE_REPLACE = 'replace';
|
||||
|
||||
/** @var array<string, true> folders of the current restore already cleared of stale temporaries */
|
||||
private array $cleaned = [];
|
||||
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private IRootFolder $root,
|
||||
@@ -58,7 +61,8 @@ final class RestoreService {
|
||||
$prefix = trim($path, '/');
|
||||
$prefix = $prefix === '' ? '' : $prefix . '/';
|
||||
$children = [];
|
||||
foreach ($this->targets->repository($target)->entries($snapshotId) as $e) {
|
||||
$repo = $this->targets->repository($target);
|
||||
foreach ($repo->entries($snapshotId) as $e) {
|
||||
if ($prefix !== '' && !str_starts_with($e['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
@@ -72,6 +76,12 @@ final class RestoreService {
|
||||
}
|
||||
$children[$name] = $c;
|
||||
}
|
||||
foreach ($repo->emptyDirs($snapshotId) as [$dir]) { // read along with the entries
|
||||
if ($prefix === '' || str_starts_with($dir, $prefix)) {
|
||||
$name = explode('/', substr($dir, strlen($prefix)), 2)[0];
|
||||
$children[$name] ??= ['name' => $name, 'type' => 'dir', 'size' => 0, 'mtime' => null, 'files' => 0];
|
||||
}
|
||||
}
|
||||
ksort($children, SORT_STRING);
|
||||
return array_values($children);
|
||||
}
|
||||
@@ -81,8 +91,8 @@ final class RestoreService {
|
||||
*
|
||||
* @return array{restored:int, trashed:int, target:string}
|
||||
*/
|
||||
public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array {
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress) {
|
||||
public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null, ?array &$skipped = null): array {
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress, &$skipped) {
|
||||
$last = time();
|
||||
// Keep the lease alive while files are being written.
|
||||
$beat = function (int $done, int $total, string $name) use ($refresh, $progress, &$last): void {
|
||||
@@ -94,11 +104,12 @@ final class RestoreService {
|
||||
$progress($done, $total, $name);
|
||||
}
|
||||
};
|
||||
return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat);
|
||||
return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat, $skipped);
|
||||
});
|
||||
}
|
||||
|
||||
private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress): array {
|
||||
/** @param array<string, string>|null $skipped see Repository::restore() */
|
||||
private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress, ?array &$skipped): array {
|
||||
if (!Repository::isSafePath($logicalPath) || !preg_match('#^data/([^/]+)/files(?:/(.*))?$#', $logicalPath, $m)) {
|
||||
throw new \InvalidArgumentException('Expected a path like data/<user>/files/<folder or file>');
|
||||
}
|
||||
@@ -119,7 +130,17 @@ final class RestoreService {
|
||||
$entries[substr($e['p'], strlen($source) + 1)] = $e;
|
||||
}
|
||||
}
|
||||
if ($entries === []) {
|
||||
// Folders without anything in them, relative to $source ('' when $source itself is one).
|
||||
/** @var array<array-key, int> $emptyDirs modification times; a folder called "2026" is an integer key */
|
||||
$emptyDirs = [];
|
||||
foreach ($single ? [] : $repo->emptyDirs($snapshotId) as [$dir, $mtime]) {
|
||||
if ($dir === $source) {
|
||||
$emptyDirs[''] = $mtime;
|
||||
} elseif (str_starts_with($dir, $source . '/')) {
|
||||
$emptyDirs[substr($dir, strlen($source) + 1)] = $mtime;
|
||||
}
|
||||
}
|
||||
if ($entries === [] && $emptyDirs === []) {
|
||||
throw new NotFoundException("Nothing at $source in snapshot $snapshotId");
|
||||
}
|
||||
|
||||
@@ -130,21 +151,62 @@ final class RestoreService {
|
||||
} elseif ($mode !== self::MODE_MERGE && $mode !== self::MODE_REPLACE) {
|
||||
throw new \InvalidArgumentException("Unknown mode $mode");
|
||||
}
|
||||
$destExisted = $parentRel === '' || $userFolder->nodeExists($parentRel);
|
||||
$dest = $this->ensureFolder($userFolder, $parentRel);
|
||||
$names = $single ? [basename($rel) => $entries['']] : $entries;
|
||||
|
||||
// The folders with something in them that this restore creates get their time back; one that
|
||||
// was already there keeps its own, as its content changes now. Noted before anything is written.
|
||||
/** @var array<array-key, int> $newFolders a folder called "2026" is an integer key */
|
||||
$newFolders = [];
|
||||
foreach ($single ? [] : $repo->folderTimes($snapshotId) as [$dir, $mtime]) {
|
||||
$relative = $dir === $source ? '' : (str_starts_with($dir, $source . '/') ? substr($dir, strlen($source) + 1) : null);
|
||||
if ($relative !== null && ($relative === '' ? !$destExisted : !$dest->nodeExists($relative))) {
|
||||
$newFolders[$relative] = $mtime;
|
||||
}
|
||||
}
|
||||
|
||||
$this->cleaned = [];
|
||||
$restored = 0;
|
||||
foreach ($names as $relPath => $entry) {
|
||||
$this->writeFile($repo, $dest, (string)$relPath, $entry);
|
||||
try {
|
||||
$this->writeFile($repo, $dest, (string)$relPath, $entry);
|
||||
} catch (\OCA\NgBackup\Repository\RepositoryException|\OCA\NgBackup\Backend\BackendException|\OCA\NgBackup\Crypto\CryptoException $e) {
|
||||
// Damaged or missing on the location (writeFile removed its temporary)
|
||||
if ($skipped === null) {
|
||||
throw $e;
|
||||
}
|
||||
$skipped[$entry['p']] = $e->getMessage();
|
||||
continue;
|
||||
}
|
||||
$restored++;
|
||||
if ($progress !== null) {
|
||||
$progress($restored, count($names), (string)$relPath);
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($emptyDirs as $dir => $mtime) {
|
||||
$dir = (string)$dir;
|
||||
$existed = $dir === '' ? $destExisted : $dest->nodeExists($dir); // one that is there keeps its own time
|
||||
$folder = $this->ensureFolder($dest, $dir);
|
||||
if (!$existed) {
|
||||
$folder->touch($mtime);
|
||||
}
|
||||
if ($progress !== null) {
|
||||
$progress($restored, count($names), $dir); // keeps the lease alive too
|
||||
}
|
||||
}
|
||||
|
||||
$trashed = 0;
|
||||
if ($mode === self::MODE_REPLACE && !$single) {
|
||||
$trashed = $this->trashExtras($dest, '', array_flip(array_keys($names)));
|
||||
$trashed = $this->trashExtras($dest, '', array_flip(array_keys($names)) + $emptyDirs);
|
||||
}
|
||||
// Last and deepest first: a change in a folder also touches the folders above it.
|
||||
foreach (array_reverse($newFolders, true) as $dir => $mtime) {
|
||||
$this->ensureFolder($dest, (string)$dir)->touch($mtime);
|
||||
if ($progress !== null) {
|
||||
$progress($restored, count($names), (string)$dir); // keeps the lease alive too
|
||||
}
|
||||
}
|
||||
return ['restored' => $restored, 'trashed' => $trashed, 'target' => $uid . ':' . $userFolder->getRelativePath($dest->getPath())];
|
||||
}
|
||||
@@ -155,15 +217,18 @@ final class RestoreService {
|
||||
* @param bool $stripPrefix write into $directory itself instead of $directory/$prefix (see
|
||||
* Repository::restore()); used to restore "data/" or "config/" straight into the
|
||||
* real data/config directory for occ backup:restore:full.
|
||||
* @param bool $keepUnchanged leave files already in place with the snapshot's size and
|
||||
* modification time (see Repository::restore()): a full restore that runs again
|
||||
* goes on where it stopped
|
||||
*/
|
||||
public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null): int {
|
||||
public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null, ?array &$skipped = null, bool $keepUnchanged = false): int {
|
||||
if ($prefix !== '' && !Repository::isSafePath($prefix)) {
|
||||
throw new \InvalidArgumentException('Invalid path: ' . $prefix);
|
||||
}
|
||||
if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) {
|
||||
throw new \RuntimeException("Cannot create $directory");
|
||||
throw new \OCA\NgBackup\Repository\RestoreTargetException("Cannot create $directory (disk full, or no permission?)");
|
||||
}
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat): int {
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat, &$skipped, $keepUnchanged): int {
|
||||
$own = LeaseService::throttled($refresh);
|
||||
$beat = static function () use ($own, $heartbeat): void {
|
||||
$own();
|
||||
@@ -171,13 +236,17 @@ final class RestoreService {
|
||||
$heartbeat();
|
||||
}
|
||||
};
|
||||
return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat);
|
||||
return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat, $skipped, $keepUnchanged);
|
||||
}, 3600);
|
||||
}
|
||||
|
||||
private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void {
|
||||
$dir = dirname($relPath);
|
||||
$folder = $dir === '.' ? $dest : $this->ensureFolder($dest, $dir);
|
||||
if (!isset($this->cleaned[$folder->getPath()])) {
|
||||
$this->removeStaleTemporaries($folder);
|
||||
$this->cleaned[$folder->getPath()] = true;
|
||||
}
|
||||
$name = basename($relPath);
|
||||
// Stage under a temporary name; only rename into place after every blob was verified.
|
||||
$tmpName = '.' . $name . '.ngb-restore-' . bin2hex(random_bytes(3));
|
||||
@@ -195,7 +264,7 @@ final class RestoreService {
|
||||
throw new \RuntimeException('Write failed for ' . $relPath);
|
||||
}
|
||||
if ($written !== $entry['s']) {
|
||||
throw new \RuntimeException("Size mismatch restoring $relPath");
|
||||
throw new \OCA\NgBackup\Repository\RepositoryException("Size mismatch restoring $relPath");
|
||||
}
|
||||
if ($folder->nodeExists($name)) {
|
||||
$existing = $folder->get($name);
|
||||
@@ -223,12 +292,25 @@ final class RestoreService {
|
||||
}
|
||||
}
|
||||
|
||||
/** See Repository::removeStaleTemporaries(): what a restore that was stopped left in this folder. */
|
||||
private function removeStaleTemporaries(Folder $folder): void {
|
||||
foreach ($folder->getDirectoryListing() as $node) {
|
||||
if ($node instanceof File && Repository::isRestoreTemporary($node->getName())) {
|
||||
try {
|
||||
$node->delete();
|
||||
} catch (\Throwable) {
|
||||
// in the way of nothing: the new temporary gets a name of its own
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function trashExtras(Folder $folder, string $prefix, array $keep): int {
|
||||
$count = 0;
|
||||
foreach ($folder->getDirectoryListing() as $node) {
|
||||
$rel = $prefix === '' ? $node->getName() : $prefix . '/' . $node->getName();
|
||||
if ($node instanceof Folder) {
|
||||
$hasKept = false;
|
||||
$hasKept = isset($keep[$rel]); // a folder that was empty in the snapshot
|
||||
foreach (array_keys($keep) as $k) {
|
||||
if (str_starts_with((string)$k, $rel . '/')) {
|
||||
$hasKept = true;
|
||||
|
||||
@@ -28,6 +28,19 @@ final class RetentionPolicy {
|
||||
return ['last' => $this->last, 'daily' => $this->daily, 'weekly' => $this->weekly, 'monthly' => $this->monthly];
|
||||
}
|
||||
|
||||
/**
|
||||
* keep() for the snapshots that are not pinned, plus the pinned ones. A pinned snapshot does not
|
||||
* take the place of a daily, weekly or monthly one.
|
||||
*
|
||||
* @param array<string, int> $snapshots id => unix time
|
||||
* @param list<string> $pinned
|
||||
* @return array<string, list<string>>
|
||||
*/
|
||||
public function keepWithPinned(array $snapshots, array $pinned): array {
|
||||
$pins = array_intersect_key($snapshots, array_flip($pinned));
|
||||
return $this->keep(array_diff_key($snapshots, $pins)) + array_fill_keys(array_keys($pins), ['pinned']);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string, int> $snapshots id => unix time
|
||||
* @return array<string, list<string>> id => reasons it is kept (ids not in the result are removed)
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\AppInfo\Application;
|
||||
use OCP\IAppConfig;
|
||||
use OCP\IConfig;
|
||||
use OCP\IGroupManager;
|
||||
|
||||
/**
|
||||
* The time zone times are shown in, and the daily time from before each location had its own
|
||||
* schedule (see LocationSchedule). Nextcloud runs PHP in UTC, so a time without a zone meant UTC
|
||||
* (issue #19: 03:00 ran at 05:00 in Berlin). A time saved before there was a zone is converted once
|
||||
* to the zone of the server or of an administrator, at today's offset: it then shows as it
|
||||
* effectively ran (02:30 UTC becomes 04:30 Europe/Berlin in summer).
|
||||
*/
|
||||
final class Schedule {
|
||||
public const TIME_KEY = 'schedule_time';
|
||||
public const ZONE_KEY = 'schedule_timezone';
|
||||
|
||||
public function __construct(
|
||||
private IAppConfig $appConfig,
|
||||
private IConfig $config,
|
||||
private IGroupManager $groups,
|
||||
) {
|
||||
}
|
||||
|
||||
/** @return array{time: string, timezone: non-empty-string} time "" = no schedule */
|
||||
public function get(): array {
|
||||
$time = $this->appConfig->getValueString(Application::APP_ID, self::TIME_KEY, '');
|
||||
$zone = $this->appConfig->getValueString(Application::APP_ID, self::ZONE_KEY, '');
|
||||
if ($zone === '' || !self::validZone($zone)) {
|
||||
$zone = $this->guessZone();
|
||||
if ($time !== '' && self::validTime($time)) {
|
||||
$time = self::convert($time, 'UTC', $zone);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::TIME_KEY, $time);
|
||||
}
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::ZONE_KEY, $zone);
|
||||
}
|
||||
return ['time' => $time, 'timezone' => $zone];
|
||||
}
|
||||
|
||||
/** @return non-empty-string the zone of a location's schedule, or else this one */
|
||||
public function zoneOr(?string $zone): string {
|
||||
return $zone !== null && self::validZone($zone) ? $zone : $this->get()['timezone'];
|
||||
}
|
||||
|
||||
/**
|
||||
* "HH:MM" in zone $from as "HH:MM" in zone $to, at today's offsets.
|
||||
*
|
||||
* @param non-empty-string $from
|
||||
* @param non-empty-string $to
|
||||
*/
|
||||
public static function convert(string $time, string $from, string $to, ?int $now = null): string {
|
||||
$day = (new \DateTimeImmutable('@' . ($now ?? time())))->setTimezone(new \DateTimeZone($from))->format('Y-m-d');
|
||||
return (new \DateTimeImmutable($day . ' ' . $time, new \DateTimeZone($from)))->setTimezone(new \DateTimeZone($to))->format('H:i');
|
||||
}
|
||||
|
||||
public static function validTime(string $time): bool {
|
||||
return preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $time) === 1;
|
||||
}
|
||||
|
||||
/** @psalm-assert-if-true non-empty-string $zone */
|
||||
public static function validZone(string $zone): bool {
|
||||
return in_array($zone, \DateTimeZone::listIdentifiers(), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* The server's default_timezone when set, else the first administrator's, else UTC. A server
|
||||
* zone of UTC is passed over on purpose: it is mostly what an image or installer left there,
|
||||
* while the administrator who set the time read it as their own (#19).
|
||||
*
|
||||
* @return non-empty-string
|
||||
*/
|
||||
private function guessZone(): string {
|
||||
$zone = $this->config->getSystemValueString('default_timezone', '');
|
||||
if ($zone !== '' && $zone !== 'UTC' && self::validZone($zone)) {
|
||||
return $zone;
|
||||
}
|
||||
foreach ($this->groups->get('admin')?->getUsers() ?? [] as $user) {
|
||||
$zone = $this->config->getUserValue($user->getUID(), 'core', 'timezone', '');
|
||||
if ($zone !== '' && self::validZone($zone)) {
|
||||
return $zone;
|
||||
}
|
||||
}
|
||||
return 'UTC';
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,9 @@ use OCP\Server;
|
||||
* classes but its enabled/disabled state is left as the admin set it.
|
||||
*/
|
||||
final class TargetService {
|
||||
/** Deletion delay taken when a location's own cannot be read: nothing leaves the Bin sooner than on the old server. */
|
||||
public const LONGEST_DELETE_DELAY = 365;
|
||||
|
||||
public function __construct(
|
||||
private TargetMapper $mapper,
|
||||
private KeyService $keys,
|
||||
@@ -35,6 +38,8 @@ final class TargetService {
|
||||
private \OCA\NgBackup\Db\DbIndexCache $indexCache,
|
||||
private \OCA\NgBackup\Db\AppConfigCatalogAnchor $anchor,
|
||||
private SnapshotMapper $snapshots,
|
||||
private AlertService $alerts,
|
||||
private \OCA\NgBackup\Db\RunMapper $runs,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -48,20 +53,34 @@ final class TargetService {
|
||||
* installation's key), then save it.
|
||||
*
|
||||
* @param array<string, mixed> $options
|
||||
* @param ?callable(int, int): void $progress reading an existing repository's index: files done, and in all
|
||||
* @return array{target: Target, created: bool}
|
||||
*/
|
||||
public function add(string $name, string $backend, string $auth, array $options, string $basePath): array {
|
||||
public function add(string $name, string $backend, string $auth, array $options, string $basePath, ?callable $progress = null, bool $repairConfig = false): array {
|
||||
$this->validateName($name);
|
||||
$this->requireUsableBackend($backend);
|
||||
$keys = $this->keys->keyRing();
|
||||
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath);
|
||||
if ($storageBackend->exists('config')) {
|
||||
$repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor);
|
||||
$created = false;
|
||||
} else {
|
||||
$repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey(), $this->anchor);
|
||||
$created = true;
|
||||
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath, !$repairConfig);
|
||||
try {
|
||||
if ($repairConfig) {
|
||||
Repository::repairConfig($storageBackend, $keys, $this->keys->wrappedKey());
|
||||
}
|
||||
if ($storageBackend->exists('config')) {
|
||||
$repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor, $progress);
|
||||
$created = false;
|
||||
} else {
|
||||
$repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey(), $this->anchor);
|
||||
$created = true;
|
||||
}
|
||||
} catch (\OCA\NgBackup\Repository\RepositoryException $e) {
|
||||
$otherKey = str_contains($e->getMessage(), 'different key') || str_contains($e->getMessage(), 'do not open with this key');
|
||||
if ($otherKey && $this->keyIsUnused()) {
|
||||
throw new \OCA\NgBackup\Repository\RepositoryException(rtrim($e->getMessage(), '.') . '. ' . self::UNUSED_KEY_HINT, 0, $e);
|
||||
}
|
||||
throw $e;
|
||||
}
|
||||
$this->keys->markUsed();
|
||||
$this->recordSettings($repo);
|
||||
|
||||
foreach ($this->mapper->findAll() as $existing) {
|
||||
if ($existing->getRepositoryId() === $repo->id()) {
|
||||
@@ -77,6 +96,11 @@ final class TargetService {
|
||||
$target->setBasePath(trim($basePath, '/'));
|
||||
$target->setRepositoryId($repo->id());
|
||||
$target->setCreatedAt(time());
|
||||
// Like the location before it: a backup scheme is usually one scheme.
|
||||
$first = $this->mapper->findAll()[0] ?? null;
|
||||
$target->setSchedule($first?->getSchedule() ?? '');
|
||||
$target->setScheduleZone($first?->getScheduleZone());
|
||||
$target->setScheduleSince(time());
|
||||
$target = $this->mapper->insert($target);
|
||||
if (!$created) {
|
||||
// Disaster recovery: occ backup:list reads this local cache, not the repository
|
||||
@@ -86,6 +110,128 @@ final class TargetService {
|
||||
return ['target' => $target, 'created' => $created];
|
||||
}
|
||||
|
||||
public const UNUSED_KEY_HINT = 'The backup key of this server was never used (no location, no backup), so it can make way for the key of these backups: '
|
||||
. 'with the old server\'s recovery kit, occ backup:key:import <kit file> --replace-unused-key; without the kit, '
|
||||
. 'occ backup:target:add with the same details and --replace-unused-key (asks for a passphrase of the old server)';
|
||||
|
||||
/**
|
||||
* A key that never protected anything: no location was created or adopted with it, and the
|
||||
* database has no location and no run (installations from before the flag existed). Only such
|
||||
* a key may make way for the key of existing backups, e.g. after a new key was made by
|
||||
* mistake on a server that is being restored.
|
||||
*/
|
||||
public function keyIsUnused(): bool {
|
||||
return $this->keys->isInitialized() && !$this->keys->wasUsed()
|
||||
&& $this->mapper->findAll() === [] && $this->runs->countAll() === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* occ backup:key:import on a server whose own key was never used: the kit's key takes its place.
|
||||
*
|
||||
* @param array<string, mixed> $kit
|
||||
* @throws \OCA\NgBackup\Crypto\CryptoException wrong passphrase: nothing changed
|
||||
*/
|
||||
public function importKitReplacingUnusedKey(array $kit, #[\SensitiveParameter] string $passphrase): void {
|
||||
$old = $this->requireUnusedKey();
|
||||
$this->keys->importRecoveryKit($kit, $passphrase, true);
|
||||
$this->alerts->securityEvent('key_replaced', ['old' => $old, 'new' => $this->keys->fingerprint()]);
|
||||
}
|
||||
|
||||
/** @return string the fingerprint of the unused key */
|
||||
private function requireUnusedKey(): string {
|
||||
if (!$this->keys->isInitialized()) {
|
||||
throw new \RuntimeException('There is no backup key here to replace');
|
||||
}
|
||||
if (!$this->keyIsUnused()) {
|
||||
throw new \RuntimeException('The backup key of this server (fingerprint ' . $this->keys->fingerprint() . ') has been used for a backup location: it is never replaced');
|
||||
}
|
||||
return $this->keys->fingerprint();
|
||||
}
|
||||
|
||||
public const LOCATION_REPOSITORY = 'repository';
|
||||
public const LOCATION_DATA_WITHOUT_CONFIG = 'data-without-config';
|
||||
public const LOCATION_EMPTY = 'empty';
|
||||
|
||||
/**
|
||||
* What these settings lead to: a repository, backup data whose config is gone, or nothing
|
||||
* of NG Backup. Creates nothing.
|
||||
*
|
||||
* @return self::LOCATION_*
|
||||
*/
|
||||
public function locationState(string $backend, string $auth, array $options, string $basePath): string {
|
||||
$this->requireUsableBackend($backend);
|
||||
$storage = $this->buildBackend($backend, $auth, $options, $basePath, false);
|
||||
if ($storage->exists('config')) {
|
||||
return self::LOCATION_REPOSITORY;
|
||||
}
|
||||
return Repository::holdsData($storage) ? self::LOCATION_DATA_WITHOUT_CONFIG : self::LOCATION_EMPTY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Disaster recovery without the recovery kit, on an installation without a backup key: take
|
||||
* the key from the location's config with one of its passphrases, then add the location as
|
||||
* usual. The key is only kept when it really is the one the repository was made with.
|
||||
* With $replaceUnusedKey, a key this server has but never used makes way (keyIsUnused()).
|
||||
*
|
||||
* @param array<string, mixed> $options
|
||||
* @param ?callable(int, int): void $progress
|
||||
* @return array{target: Target, created: bool, deleteDelayDays: ?int, settingsUnreadable: bool} the deletion delay the location had stored, if any, and whether its settings could not be read
|
||||
* @throws \OCA\NgBackup\Crypto\CryptoException wrong passphrase
|
||||
*/
|
||||
public function addWithPassphrase(string $name, string $backend, string $auth, array $options, string $basePath,
|
||||
#[\SensitiveParameter] string $passphrase, ?callable $progress = null, bool $replaceUnusedKey = false): array {
|
||||
$old = null;
|
||||
if ($replaceUnusedKey) {
|
||||
$old = $this->requireUnusedKey();
|
||||
} elseif ($this->keys->isInitialized()) {
|
||||
throw new \RuntimeException('A backup key already exists here');
|
||||
}
|
||||
$this->validateName($name);
|
||||
$this->requireUsableBackend($backend);
|
||||
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath, false);
|
||||
if (!$storageBackend->exists('config')) {
|
||||
throw new \InvalidArgumentException("No NG Backup repository found in folder '" . trim($basePath, '/') . "'");
|
||||
}
|
||||
$wrapped = Repository::wrappedKeyAt($storageBackend);
|
||||
$keys = \OCA\NgBackup\Crypto\KeyRing::unwrapAny($wrapped, $passphrase);
|
||||
if (!Repository::madeWithKey($storageBackend, $keys)) {
|
||||
throw new \RuntimeException('The key in the config at this location does not open its backups: the config is damaged or was replaced. Only the recovery kit can help now: occ backup:key:import <kit file>, then this command again.');
|
||||
}
|
||||
$settings = Repository::settingsAt($storageBackend, $keys);
|
||||
$stored = $settings['delete_delay_days'] ?? null;
|
||||
$delay = is_int($stored) && $stored >= 1 && $stored <= 365 ? $stored : null;
|
||||
// Settings that are there but cannot be read, or hold no valid delay (also none at all): the old server's delay
|
||||
// is unknown, so take the longest one; a shorter guess could empty the Bin too soon.
|
||||
$unknown = ($settings['unreadable'] ?? false) === true || ($settings !== [] && $delay === null);
|
||||
$this->keys->adoptKey($keys, $wrapped, $delay ?? ($unknown ? self::LONGEST_DELETE_DELAY : null), $replaceUnusedKey);
|
||||
if ($old !== null) {
|
||||
$this->alerts->securityEvent('key_replaced', ['old' => $old, 'new' => $this->keys->fingerprint()]);
|
||||
}
|
||||
return $this->add($name, $backend, $auth, $options, $basePath, $progress) + ['deleteDelayDays' => $delay, 'settingsUnreadable' => $unknown];
|
||||
}
|
||||
|
||||
/** A passphrase change that missed this location (offline at the time) reaches it now. A failure is tried again at the next backup. */
|
||||
public function catchUpWrappedKey(Target $target, Repository $repo): void {
|
||||
if (!$this->keys->wrappedKeyPending($target->getId())) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
$repo->updateWrappedKey($this->keys->wrappedKey());
|
||||
$this->keys->setWrappedKeyPending($target->getId(), false);
|
||||
} catch (\Throwable $e) {
|
||||
Server::get(\Psr\Log\LoggerInterface::class)->warning('NG Backup: the last passphrase change still does not reach ' . $target->getName()
|
||||
. ' (it keeps the previous passphrases); tried again at its next backup: ' . $e->getMessage(), ['exception' => $e]);
|
||||
}
|
||||
}
|
||||
|
||||
/** Seal the settings fixed at installation on the location, once. A failure is tried again at the next backup. */
|
||||
public function recordSettings(Repository $repo): void {
|
||||
try {
|
||||
$repo->recordSettings(['delete_delay_days' => $this->keys->deleteDelayDays()]);
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Change how an existing location is reached (credentials, host, folder, even the connection
|
||||
* type) or its name. The new settings must lead to the same repository: the history stays
|
||||
@@ -207,7 +353,25 @@ final class TargetService {
|
||||
return $result;
|
||||
}
|
||||
|
||||
/** Update the local copy of a snapshot's note or pin, if the snapshot is listed here at all. */
|
||||
public function syncSnapshot(Target $target, string $snapshotId, ?string $note = null, ?bool $pinned = null): void {
|
||||
try {
|
||||
$s = $this->snapshots->findOne($target->getId(), $snapshotId);
|
||||
} catch (DoesNotExistException) {
|
||||
return;
|
||||
}
|
||||
if ($note !== null) {
|
||||
$s->setNote($note !== '' ? $note : null);
|
||||
}
|
||||
if ($pinned !== null) {
|
||||
$s->setPinned($pinned);
|
||||
}
|
||||
$this->snapshots->update($s);
|
||||
}
|
||||
|
||||
private function adoptSnapshots(Target $target, Repository $repo): void {
|
||||
$notes = $repo->snapshotNotes();
|
||||
$pinned = array_flip($repo->pinnedSnapshots());
|
||||
foreach ($repo->listSnapshots() as $snapshotId) {
|
||||
try {
|
||||
$this->snapshots->findOne($target->getId(), $snapshotId);
|
||||
@@ -226,8 +390,11 @@ final class TargetService {
|
||||
$s->setCreatedAt((int)strtotime((string)($meta['time'] ?? 'now')));
|
||||
$s->setFiles((int)($meta['stats']['files'] ?? 0));
|
||||
$s->setBytes((int)($meta['stats']['bytes'] ?? 0));
|
||||
$s->setNote($notes[$snapshotId] ?? null);
|
||||
$s->setPinned(isset($pinned[$snapshotId]));
|
||||
$this->snapshots->insert($s);
|
||||
} catch (\Throwable) {
|
||||
} catch (\Throwable $e) {
|
||||
Server::get(\Psr\Log\LoggerInterface::class)->warning('NG Backup: snapshot ' . $snapshotId . ' on ' . $target->getName() . ' not listed: ' . $e->getMessage(), ['exception' => $e]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -246,6 +413,30 @@ final class TargetService {
|
||||
}
|
||||
|
||||
/** Size limit for the data stored on this location (null = none). */
|
||||
/**
|
||||
* @param string $schedule see LocationSchedule
|
||||
* @throws \InvalidArgumentException
|
||||
*/
|
||||
public function setSchedule(Target $target, string $schedule, string $zone): Target {
|
||||
if (!Schedule::validZone($zone)) {
|
||||
throw new \InvalidArgumentException('Unknown time zone: ' . $zone);
|
||||
}
|
||||
$new = LocationSchedule::normalize($schedule);
|
||||
$turnedOff = $new === '' && (string)$target->getSchedule() !== '';
|
||||
if ($new === (string)$target->getSchedule() && $zone === $target->getScheduleZone()) {
|
||||
// Saved again unchanged: a moment that already passed still gets its backup
|
||||
return $target;
|
||||
}
|
||||
$target->setSchedule($new);
|
||||
$target->setScheduleZone($zone);
|
||||
$target->setScheduleSince(time());
|
||||
$target = $this->mapper->update($target);
|
||||
if ($turnedOff) {
|
||||
$this->alerts->securityEvent('schedule_off', ['target' => $target->getName()]);
|
||||
}
|
||||
return $target;
|
||||
}
|
||||
|
||||
public function setMaxBytes(Target $target, ?int $maxBytes): Target {
|
||||
$target->setMaxBytes($maxBytes);
|
||||
return $this->mapper->update($target);
|
||||
@@ -259,6 +450,30 @@ final class TargetService {
|
||||
/** Remove the target from NG Backup. The backups on the target itself are not touched. */
|
||||
public function remove(Target $target): void {
|
||||
$this->mapper->delete($target);
|
||||
// The location is gone now: tidying up must not fail the removal, or the caller would skip
|
||||
// the removal event and a retry would no longer find the location
|
||||
$this->tidyUp($target, fn () => $this->keys->setWrappedKeyPending($target->getId(), false));
|
||||
// The cached index of its repository is only of use while a location points at it
|
||||
$this->tidyUp($target, function () use ($target): void {
|
||||
$repositoryId = $target->getRepositoryId();
|
||||
if ($repositoryId !== null && !array_filter($this->list(), fn (Target $t) => $t->getRepositoryId() === $repositoryId)) {
|
||||
$this->indexCache->forget($repositoryId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** One tidy-up step after a removal: a failure is logged (if even that works) and never thrown. */
|
||||
private function tidyUp(Target $target, callable $step): void {
|
||||
try {
|
||||
$step();
|
||||
} catch (\Throwable $e) {
|
||||
try {
|
||||
Server::get(\Psr\Log\LoggerInterface::class)->warning('NG Backup: location ' . $target->getName()
|
||||
. ' removed, tidying up after it failed: ' . $e->getMessage(), ['exception' => $e]);
|
||||
} catch (\Throwable) {
|
||||
// nothing left to tell it to
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function backend(Target $target): IBackend {
|
||||
@@ -281,7 +496,10 @@ final class TargetService {
|
||||
|
||||
private function buildBackend(string $backend, string $auth, array $options, string $basePath, bool $createBase = true): StorageBackend {
|
||||
$create = fn (): IStorage => $this->factory()->create($backend, $auth, $options);
|
||||
return new StorageBackend($create(), $basePath, $createBase, $create);
|
||||
$storage = new StorageBackend($create(), $basePath, $createBase, $create);
|
||||
$storage->readAheadConnections = Server::get(\OCP\IAppConfig::class)
|
||||
->getValueInt('ng_backup', 'restore_sftp_connections', \OCA\NgBackup\Backend\SftpRangeFetcher::DEFAULT_CONNECTIONS);
|
||||
return $storage;
|
||||
}
|
||||
|
||||
private ?ExternalStorageFactory $factory = null;
|
||||
|
||||
@@ -110,11 +110,6 @@
|
||||
<code><![CDATA[ask]]></code>
|
||||
</UndefinedInterfaceMethod>
|
||||
</file>
|
||||
<file src="lib/Command/Trash.php">
|
||||
<RedundantCastGivenDocblockType>
|
||||
<code><![CDATA[(string)($info['time'] ?? '')]]></code>
|
||||
</RedundantCastGivenDocblockType>
|
||||
</file>
|
||||
<file src="lib/Controller/AdminApiController.php">
|
||||
<InvalidOperand>
|
||||
<code><![CDATA[$maxGb * 1073741824]]></code>
|
||||
@@ -153,10 +148,6 @@
|
||||
</PossiblyFalseArgument>
|
||||
</file>
|
||||
<file src="lib/Db/DbDumper.php">
|
||||
<InvalidArrayOffset>
|
||||
<code><![CDATA[$info['sha256']]]></code>
|
||||
<code><![CDATA[$manifest['sequences']]]></code>
|
||||
</InvalidArrayOffset>
|
||||
<MissingClosureReturnType>
|
||||
<code><![CDATA[fn ($c) => $end[array_search($c, $columns, true)]]]></code>
|
||||
<code><![CDATA[fn ($c) => $q($c)]]></code>
|
||||
@@ -189,6 +180,13 @@
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[$this->db]]></code>
|
||||
<code><![CDATA[TransactionIsolationLevel]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\ParameterType]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\ParameterType]]></code>
|
||||
@@ -196,6 +194,11 @@
|
||||
<code><![CDATA[\Doctrine\DBAL\Platforms\PostgreSQLPlatform]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Platforms\PostgreSQLPlatform]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Platforms\PostgreSQLPlatform]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Platforms\PostgreSQLPlatform]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Schema\ForeignKeyConstraint]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
||||
@@ -297,6 +300,22 @@
|
||||
<code><![CDATA[$t]]></code>
|
||||
</UndefinedDocblockClass>
|
||||
</file>
|
||||
<file src="lib/Migration/Version000008Date20261107010000.php">
|
||||
<UndefinedDocblockClass>
|
||||
<code><![CDATA[$t]]></code>
|
||||
<code><![CDATA[$t]]></code>
|
||||
<code><![CDATA[$t]]></code>
|
||||
<code><![CDATA[$t]]></code>
|
||||
</UndefinedDocblockClass>
|
||||
</file>
|
||||
<file src="lib/Migration/Version000007Date20261107000000.php">
|
||||
<UndefinedDocblockClass>
|
||||
<code><![CDATA[$t]]></code>
|
||||
<code><![CDATA[$t]]></code>
|
||||
<code><![CDATA[$t]]></code>
|
||||
<code><![CDATA[$t]]></code>
|
||||
</UndefinedDocblockClass>
|
||||
</file>
|
||||
<file src="lib/Migration/Version000005Date20261002040000.php">
|
||||
<UndefinedDocblockClass>
|
||||
<code><![CDATA[$t]]></code>
|
||||
@@ -348,7 +367,6 @@
|
||||
<code><![CDATA[$encrypted]]></code>
|
||||
<code><![CDATA[$encrypted]]></code>
|
||||
<code><![CDATA[$encrypted]]></code>
|
||||
<code><![CDATA[$fh]]></code>
|
||||
<code><![CDATA[$plain]]></code>
|
||||
<code><![CDATA[$plain]]></code>
|
||||
<code><![CDATA[$plain]]></code>
|
||||
@@ -438,6 +456,7 @@
|
||||
</EmptyArrayAccess>
|
||||
<UndefinedInterfaceMethod>
|
||||
<code><![CDATA[getInner]]></code>
|
||||
<code><![CDATA[getInner]]></code>
|
||||
</UndefinedInterfaceMethod>
|
||||
<UnresolvableInclude>
|
||||
<code><![CDATA[include $path]]></code>
|
||||
|
||||
|
Before Width: | Height: | Size: 226 KiB After Width: | Height: | Size: 244 KiB |
|
Before Width: | Height: | Size: 254 KiB After Width: | Height: | Size: 228 KiB |
|
Before Width: | Height: | Size: 227 KiB After Width: | Height: | Size: 222 KiB |
|
After Width: | Height: | Size: 172 KiB |
@@ -7,20 +7,41 @@
|
||||
<NcSettingsSection :name="t('ng_backup', 'NG Backup')"
|
||||
:description="t('ng_backup', 'Encrypted, incremental backups of this Nextcloud (database, configuration and data). Backup locations are never mounted and are not visible in Files.')">
|
||||
<p class="ngb-feedback">
|
||||
{{ t('ng_backup', 'NG Backup is in beta. We would like to hear how it works for you, also when everything went fine.') }}
|
||||
{{ t('ng_backup', 'We would like to hear how NG Backup works for you, also when everything went fine.') }}
|
||||
<a :href="feedbackUrl" target="_blank" rel="noopener noreferrer">{{ t('ng_backup', 'Share your experience ↗') }}</a>
|
||||
</p>
|
||||
<NcLoadingIcon v-if="!status" :size="32" />
|
||||
<NcNoteCard v-if="error" type="error">{{ error }}</NcNoteCard>
|
||||
<NcNoteCard v-if="status && !status.cron.ok" type="warning">
|
||||
<p>
|
||||
{{ status.cron.stopped
|
||||
? t('ng_backup', 'Background jobs are set to system cron, but cron has not run since {time}.', { time: lastCron })
|
||||
: t('ng_backup', 'Background jobs do not run on a system cron (they are set to "{mode}").', { mode: status.cron.mode === 'webcron' ? 'Webcron' : 'AJAX' }) }}
|
||||
{{ t('ng_backup', 'Backups need a system cron that runs every 5 minutes, also the scheduled ones.') }}
|
||||
<a :href="status.cron.docs" target="_blank" rel="noopener noreferrer">{{ t('ng_backup', 'How to set up cron ↗') }}</a>
|
||||
</p>
|
||||
<p>{{ t('ng_backup', 'Until then, a backup started with "Back up now" only moves on while this page is open.') }}</p>
|
||||
</NcNoteCard>
|
||||
</NcSettingsSection>
|
||||
|
||||
<template v-if="status">
|
||||
<div class="ngb-tabs">
|
||||
<NcButton :variant="tab === 'backup' ? 'primary' : 'secondary'" @click="tab = 'backup'">{{ t('ng_backup', 'Backup and restore') }}</NcButton>
|
||||
<NcButton v-if="status.key.everConfirmed && status.targets.length" :variant="tab === 'bin' ? 'primary' : 'secondary'" @click="tab = 'bin'">{{ t('ng_backup', 'Bin') }}</NcButton>
|
||||
<NcButton :variant="tab === 'settings' ? 'primary' : 'secondary'" @click="tab = 'settings'">{{ t('ng_backup', 'Settings') }}</NcButton>
|
||||
<NcButton :variant="tab === 'docs' ? 'primary' : 'secondary'" @click="tab = 'docs'">{{ t('ng_backup', 'Docs') }}</NcButton>
|
||||
</div>
|
||||
|
||||
<template v-if="tab === 'settings'">
|
||||
<NcSettingsSection v-if="tab === 'docs'" :name="t('ng_backup', 'Documentation')">
|
||||
<ul class="ngb-docs">
|
||||
<li v-for="doc in docs" :key="doc.url">
|
||||
<a :href="doc.url" target="_blank" rel="noopener noreferrer">{{ doc.name }} ↗</a>
|
||||
<span>{{ doc.description }}</span>
|
||||
</li>
|
||||
</ul>
|
||||
</NcSettingsSection>
|
||||
<Bin v-else-if="tab === 'bin' && status.key.everConfirmed && status.targets.length" :status="status" @changed="refresh" />
|
||||
<template v-else-if="tab === 'settings'">
|
||||
<KeySetup :status="status" @changed="refresh" />
|
||||
<Locations v-if="status.key.everConfirmed" :status="status" @changed="refresh" />
|
||||
</template>
|
||||
@@ -37,7 +58,7 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onBeforeUnmount, onMounted, ref } from 'vue'
|
||||
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
|
||||
import { t } from '@nextcloud/l10n'
|
||||
import NcButton from '@nextcloud/vue/components/NcButton'
|
||||
import NcLoadingIcon from '@nextcloud/vue/components/NcLoadingIcon'
|
||||
@@ -45,6 +66,7 @@ import NcNoteCard from '@nextcloud/vue/components/NcNoteCard'
|
||||
import NcSettingsSection from '@nextcloud/vue/components/NcSettingsSection'
|
||||
import api from './api.js'
|
||||
import Backups from './components/Backups.vue'
|
||||
import Bin from './components/Bin.vue'
|
||||
import KeySetup from './components/KeySetup.vue'
|
||||
import Locations from './components/Locations.vue'
|
||||
import RestoreBrowser from './components/RestoreBrowser.vue'
|
||||
@@ -52,6 +74,25 @@ import RestoreBrowser from './components/RestoreBrowser.vue'
|
||||
// Opens the experience report form in .github/ISSUE_TEMPLATE
|
||||
const feedbackUrl = 'https://github.com/dillardblom/ng_backup/issues/new?template=experience.yml'
|
||||
|
||||
const repoUrl = 'https://github.com/dillardblom/ng_backup'
|
||||
const docs = [
|
||||
{
|
||||
name: t('ng_backup', 'How-to'),
|
||||
description: t('ng_backup', 'Step by step: from first install to a tested restore.'),
|
||||
url: repoUrl + '/blob/main/docs/HOWTO.md',
|
||||
},
|
||||
{
|
||||
name: t('ng_backup', 'Disaster recovery'),
|
||||
description: t('ng_backup', 'Your server is gone: get Nextcloud back on a new server from a backup location.'),
|
||||
url: repoUrl + '/blob/main/docs/DISASTER-RECOVERY.md',
|
||||
},
|
||||
{
|
||||
name: t('ng_backup', 'Roadmap'),
|
||||
description: t('ng_backup', 'What is planned for the next versions.'),
|
||||
url: repoUrl + '#roadmap',
|
||||
},
|
||||
]
|
||||
|
||||
const status = ref(null)
|
||||
const allRuns = ref(false) // runs older than the last days too
|
||||
const error = ref('')
|
||||
@@ -77,20 +118,36 @@ async function refresh() {
|
||||
} catch (e) {
|
||||
error.value = e.message
|
||||
}
|
||||
// Poll while something runs, so progress shows up without reloading.
|
||||
// Poll while something runs, so progress shows up without reloading. Without a working
|
||||
// system cron, each poll also lets a running backup take a short step (see tick()).
|
||||
clearTimeout(timer)
|
||||
if (status.value?.runs.some(r => r.status === 'running')) {
|
||||
timer = setTimeout(refresh, 4000)
|
||||
timer = setTimeout(tickAndRefresh, 4000)
|
||||
}
|
||||
}
|
||||
|
||||
async function tickAndRefresh() {
|
||||
if (status.value && !status.value.cron.ok && status.value.runs.some(r => r.status === 'running' && r.kind === 'full')) {
|
||||
try {
|
||||
await api.tick()
|
||||
} catch (e) {
|
||||
// The next poll tries again; the error of a failed step shows on its run.
|
||||
}
|
||||
}
|
||||
await refresh()
|
||||
}
|
||||
|
||||
const lastCron = computed(() => status.value?.cron.lastRun ? new Date(status.value.cron.lastRun * 1000).toLocaleString() : '')
|
||||
|
||||
onMounted(refresh)
|
||||
onBeforeUnmount(() => clearTimeout(timer))
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
/* NcSettingsSection sets its own width (at most 900px): the runs table needs more */
|
||||
.ng-backup :deep(.settings-section) {
|
||||
max-width: 900px;
|
||||
width: min(1400px, 100% - var(--default-grid-baseline) * 7 * 2);
|
||||
max-width: none;
|
||||
}
|
||||
.ngb-feedback {
|
||||
color: var(--color-text-maxcontrast);
|
||||
@@ -100,6 +157,17 @@ onBeforeUnmount(() => clearTimeout(timer))
|
||||
color: var(--color-primary-element);
|
||||
text-decoration: underline;
|
||||
}
|
||||
.ngb-docs li {
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.ngb-docs a {
|
||||
color: var(--color-primary-element);
|
||||
text-decoration: underline;
|
||||
margin-right: 8px;
|
||||
}
|
||||
.ngb-docs span {
|
||||
color: var(--color-text-maxcontrast);
|
||||
}
|
||||
.ngb-tabs {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
|
||||
@@ -21,7 +21,7 @@ export default {
|
||||
addSlot: (passphrase, label) => call('post', '/key/slots', { passphrase, label }),
|
||||
replaceSlot: (slot, passphrase, label) => call('put', `/key/slots/${slot}`, { passphrase, label }),
|
||||
removeSlot: (slot) => call('delete', `/key/slots/${slot}`),
|
||||
trash: (targetId) => call('get', `/targets/${targetId}/trash`),
|
||||
trash: (targetId, after = '') => call('get', `/targets/${targetId}/trash`, { after }),
|
||||
untrash: (targetId, snapshotId) => call('post', `/targets/${targetId}/trash/${snapshotId}`),
|
||||
kitUrl: () => url('/key/kit'),
|
||||
confirmKit: (accepted, phrase, code) => call('post', '/key/confirm', { accepted, phrase, code }),
|
||||
@@ -33,8 +33,12 @@ export default {
|
||||
testTarget: (id) => call('post', `/targets/${id}/test`),
|
||||
setTargetLimit: (id, maxGb) => call('put', `/targets/${id}/limit`, { maxGb }),
|
||||
startBackup: (targetId, label) => call('post', '/backups', { targetId, label }),
|
||||
setSchedule: (time) => call('put', '/schedule', { time }),
|
||||
tick: () => call('post', '/tick'),
|
||||
setPauseHeavyJobs: (enabled) => call('put', '/settings/pause-heavy-jobs', { enabled }),
|
||||
setTargetSchedule: (id, schedule, timezone) => call('put', `/targets/${id}/schedule`, { schedule, timezone }),
|
||||
snapshots: (targetId) => call('get', `/targets/${targetId}/snapshots`),
|
||||
setSnapshotNote: (targetId, snapshotId, note) => call('put', `/targets/${targetId}/snapshots/${snapshotId}/note`, { note }),
|
||||
setSnapshotPinned: (targetId, snapshotId, pinned) => call('put', `/targets/${targetId}/snapshots/${snapshotId}/pin`, { pinned }),
|
||||
browse: (targetId, snapshotId, path) => call('get', `/targets/${targetId}/snapshots/${snapshotId}/browse`, { path }),
|
||||
startRestore: (targetId, snapshotId, path, mode) => call('post', '/restores', { targetId, snapshotId, path, mode }),
|
||||
}
|
||||
|
||||
@@ -5,31 +5,20 @@
|
||||
<template>
|
||||
<NcSettingsSection :name="t('ng_backup', 'Backups')"
|
||||
:description="t('ng_backup', 'Backups run in the background through Nextcloud\'s cron. Unchanged files are not read again; only new data is uploaded.')">
|
||||
<LocationScheduleRow v-for="target in status.targets" :key="target.id" :target="target" @changed="emit('changed')" @error="error = $event" />
|
||||
<NcCheckboxRadioSwitch :model-value="status.heavyJobs.enabled" @update:model-value="savePause">
|
||||
{{ t('ng_backup', 'Pause heavy background jobs (Memories, Recognize, Preview Generator, full text search, antivirus scan) while a backup runs, at most 6 hours in a row') }}
|
||||
</NcCheckboxRadioSwitch>
|
||||
<p v-if="status.heavyJobs.jobs.length" class="ngb-muted">{{ t('ng_backup', 'Paused now: {jobs}', { jobs: status.heavyJobs.jobs.join(', ') }) }}</p>
|
||||
<div class="ngb-row">
|
||||
<NcTextField v-model="schedule" class="ngb-time" :label="t('ng_backup', 'Daily at (HH:MM, empty = off)')" />
|
||||
<NcButton :disabled="schedule === status.schedule" @click="saveSchedule">{{ t('ng_backup', 'Save') }}</NcButton>
|
||||
</div>
|
||||
<div class="ngb-row">
|
||||
<NcTextField v-model="label" class="ngb-label" :maxlength="100" :label="t('ng_backup', 'Label (optional)')"
|
||||
:placeholder="t('ng_backup', 'e.g. before the upgrade to Nextcloud 35')" />
|
||||
<NcButton v-for="target in status.targets" :key="target.id" :disabled="isRunning(target.id)" @click="start(target)">
|
||||
{{ isRunning(target.id) ? t('ng_backup', 'Backup to {name} running …', { name: target.name }) : t('ng_backup', 'Back up now to {name}', { name: target.name }) }}
|
||||
</NcButton>
|
||||
</div>
|
||||
<NcNoteCard v-if="error" type="error">{{ error }}</NcNoteCard>
|
||||
|
||||
<template v-for="target in status.targets" :key="'trash' + target.id">
|
||||
<div v-if="trash[target.id]?.length" class="ngb-trash">
|
||||
<h3>{{ t('ng_backup', 'Trash of {name}', { name: target.name }) }}</h3>
|
||||
<p class="ngb-muted">{{ t('ng_backup', 'Removed restore points can be brought back until they are deleted permanently.') }}</p>
|
||||
<ul>
|
||||
<li v-for="item in trash[target.id]" :key="item.id">
|
||||
{{ item.time ? formatDateTime(item.time) : item.id }}{{ item.label ? ' – ' + item.label : '' }}
|
||||
<span class="ngb-muted">{{ t('ng_backup', 'removed by {by}, deleted after {date}', { by: item.by, date: formatDate(item.deleteAfter) }) }}</span>
|
||||
<NcButton variant="tertiary" @click="untrash(target, item)">{{ t('ng_backup', 'Restore') }}</NcButton>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<div v-if="status.runs.length" class="ngb-table-wrap">
|
||||
<table class="ngb-table">
|
||||
<thead>
|
||||
@@ -49,11 +38,11 @@
|
||||
<td>{{ formatDateTime(run.startedAt) }}</td>
|
||||
<td>{{ targetName(run.targetId) }}</td>
|
||||
<td>{{ run.kind }}</td>
|
||||
<td>{{ run.snapshot ? formatDateTime(run.snapshot.createdAt) + (run.snapshot.label ? ' – ' + run.snapshot.label : '') : '' }}</td>
|
||||
<td :class="'ngb-' + run.status">{{ run.status }}{{ run.status === 'running' ? ' (' + run.phase + percent(run) + ')' : '' }}{{ run.error ? ': ' + run.error : '' }}</td>
|
||||
<td>{{ run.stats?.files?.files ?? run.stats?.restored ?? '' }}</td>
|
||||
<td>{{ run.stats?.files ? formatSize(run.stats.files.bytes) : '' }}</td>
|
||||
<td>{{ run.stats?.files ? formatSize(run.stats.files.uploaded) : '' }}</td>
|
||||
<td class="ngb-point">{{ run.snapshot ? (run.snapshot.pinned ? '📌 ' : '') + formatDateTime(run.snapshot.createdAt) + (run.snapshot.label ? ' – ' + run.snapshot.label : '') + (run.snapshot.note ? ' – ' + run.snapshot.note : '') : '' }}</td>
|
||||
<td :class="'ngb-' + run.status">{{ runStatus(run) }}{{ run.error ? ': ' + run.error : '' }}</td>
|
||||
<td class="ngb-num">{{ run.stats?.files?.files ?? run.stats?.restored ?? '' }}</td>
|
||||
<td class="ngb-num">{{ run.stats?.files ? formatSize(run.stats.files.bytes) : '' }}</td>
|
||||
<td class="ngb-num">{{ run.stats?.files ? formatSize(run.stats.files.uploaded) : '' }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -72,66 +61,66 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, watch } from 'vue'
|
||||
import { ref } from 'vue'
|
||||
import { n, t } from '@nextcloud/l10n'
|
||||
import NcButton from '@nextcloud/vue/components/NcButton'
|
||||
import NcCheckboxRadioSwitch from '@nextcloud/vue/components/NcCheckboxRadioSwitch'
|
||||
import NcNoteCard from '@nextcloud/vue/components/NcNoteCard'
|
||||
import NcSettingsSection from '@nextcloud/vue/components/NcSettingsSection'
|
||||
import NcTextField from '@nextcloud/vue/components/NcTextField'
|
||||
import LocationScheduleRow from './LocationScheduleRow.vue'
|
||||
import api from '../api.js'
|
||||
import { formatDateTime, formatDate, formatSize } from '../format.js'
|
||||
import { confirmPassword } from '@nextcloud/password-confirmation'
|
||||
import { reactive } from 'vue'
|
||||
import { formatDateTime, formatSize } from '../format.js'
|
||||
|
||||
const props = defineProps({ status: { type: Object, required: true }, allRuns: { type: Boolean, default: false } })
|
||||
const emit = defineEmits(['changed', 'show-all-runs'])
|
||||
const schedule = ref(props.status.schedule)
|
||||
const error = ref('')
|
||||
watch(() => props.status.schedule, (s) => { schedule.value = s })
|
||||
|
||||
const trash = reactive({})
|
||||
async function loadTrash() {
|
||||
for (const target of props.status.targets) {
|
||||
try {
|
||||
trash[target.id] = await api.trash(target.id)
|
||||
} catch (e) {
|
||||
trash[target.id] = []
|
||||
}
|
||||
const label = ref('')
|
||||
|
||||
// ": 47/183 tables" in the database phase; " 68%", or " ~68%" when the total is estimated from
|
||||
// Nextcloud's file cache (first backup)
|
||||
const percent = (run) => {
|
||||
if (!run.progress) {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
watch(() => props.status.targets.map(tg => tg.id).join(','), loadTrash, { immediate: true })
|
||||
|
||||
async function untrash(target, item) {
|
||||
error.value = ''
|
||||
try {
|
||||
await confirmPassword()
|
||||
await api.untrash(target.id, item.id)
|
||||
await loadTrash()
|
||||
emit('changed')
|
||||
} catch (e) {
|
||||
error.value = e?.message || ''
|
||||
if (run.progress.totalTables) {
|
||||
return ': ' + t('ng_backup', '{done}/{total} tables', { done: run.progress.tables, total: run.progress.totalTables })
|
||||
}
|
||||
return ' ' + (run.progress.approximate ? '~' : '') + Math.floor(run.progress.fraction * 100) + '%'
|
||||
}
|
||||
// "running (db: 47/183 tables)", or "waiting for background jobs (…, last step 12 min ago)"
|
||||
const runStatus = (run) => {
|
||||
if (run.status !== 'running') {
|
||||
return run.status
|
||||
}
|
||||
if (run.queued) {
|
||||
return t('ng_backup', 'queued, after the backup that is running now')
|
||||
}
|
||||
const detail = run.phase + percent(run)
|
||||
if (run.waitingFor) {
|
||||
return t('ng_backup', 'waiting for background jobs ({detail}, last step {minutes} min ago)', { detail, minutes: Math.floor(run.waitingFor / 60) })
|
||||
}
|
||||
return run.status + ' (' + detail + ')'
|
||||
}
|
||||
|
||||
// " 68%", or " ~68%" when the total is estimated from Nextcloud's file cache (first backup)
|
||||
const percent = (run) => run.progress ? ' ' + (run.progress.approximate ? '~' : '') + Math.floor(run.progress.fraction * 100) + '%' : ''
|
||||
const isRunning = (targetId) => props.status.runs.some(r => r.targetId === targetId && r.kind === 'full' && r.status === 'running')
|
||||
const targetName = (id) => props.status.targets.find(tg => tg.id === id)?.name ?? '#' + id
|
||||
|
||||
async function start(target) {
|
||||
async function savePause(enabled) {
|
||||
error.value = ''
|
||||
try {
|
||||
await api.startBackup(target.id, '')
|
||||
await api.setPauseHeavyJobs(enabled)
|
||||
emit('changed')
|
||||
} catch (e) {
|
||||
error.value = e.message
|
||||
}
|
||||
}
|
||||
|
||||
async function saveSchedule() {
|
||||
async function start(target) {
|
||||
error.value = ''
|
||||
try {
|
||||
await api.setSchedule(schedule.value.trim())
|
||||
await api.startBackup(target.id, label.value.trim())
|
||||
label.value = ''
|
||||
emit('changed')
|
||||
} catch (e) {
|
||||
error.value = e.message
|
||||
@@ -141,13 +130,14 @@ async function saveSchedule() {
|
||||
|
||||
<style scoped>
|
||||
.ngb-row { display: flex; gap: 8px; align-items: flex-end; flex-wrap: wrap; margin-bottom: 12px; }
|
||||
.ngb-time { max-width: 260px; }
|
||||
.ngb-table-wrap { overflow-x: auto; }
|
||||
.ngb-label { max-width: 360px; }
|
||||
.ngb-table-wrap { max-width: 90%; overflow-x: auto; } /* scrolls only on a really narrow screen */
|
||||
.ngb-table { width: 100%; border-collapse: collapse; }
|
||||
.ngb-table th, .ngb-table td { text-align: start; padding: 4px 8px; border-bottom: 1px solid var(--color-border); }
|
||||
.ngb-table th, .ngb-table td { text-align: start; padding: 4px 8px; border-bottom: 1px solid var(--color-border); white-space: normal; overflow-wrap: break-word; }
|
||||
/* Text wraps; numbers stay on one line, the restore point keeps room for its label and note */
|
||||
.ngb-table td.ngb-num { white-space: nowrap; }
|
||||
.ngb-table td.ngb-point { min-width: 9em; }
|
||||
.ngb-failed { color: var(--color-error-text); }
|
||||
.ngb-muted { color: var(--color-text-maxcontrast); }
|
||||
.ngb-trash { margin: 12px 0; }
|
||||
.ngb-trash h3 { font-weight: bold; }
|
||||
.ngb-done { color: var(--color-success-text); }
|
||||
</style>
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
<!--
|
||||
- SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
- SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
-->
|
||||
<template>
|
||||
<NcSettingsSection :name="t('ng_backup', 'Bin')"
|
||||
:description="t('ng_backup', 'Restore points that were removed, most of them by the retention policy. They can be brought back until they are deleted permanently.')">
|
||||
<NcNoteCard v-if="error" type="error">{{ error }}</NcNoteCard>
|
||||
<div v-if="reading" class="ngb-reading" role="status">
|
||||
<template v-if="reading.total === null">
|
||||
<NcLoadingIcon :size="20" />
|
||||
<span>{{ t('ng_backup', 'Connecting to {location} …', { location: reading.name }) }}</span>
|
||||
</template>
|
||||
<template v-else>
|
||||
<span>{{ t('ng_backup', 'Reading removed restore points from {location}: {done} of {total}', { location: reading.name, done: reading.done, total: reading.total }) }}</span>
|
||||
<NcProgressBar :value="reading.total ? Math.round(100 * reading.done / reading.total) : 0" />
|
||||
</template>
|
||||
<p class="ngb-muted">{{ t('ng_backup', 'This can take a while: every removed restore point is read from the location.') }}</p>
|
||||
</div>
|
||||
<NcNoteCard v-else-if="empty && !error" type="info">{{ t('ng_backup', 'The bin is empty.') }}</NcNoteCard>
|
||||
|
||||
<template v-for="target in status.targets" :key="'trash' + target.id">
|
||||
<div v-if="trash[target.id]?.length" class="ngb-trash">
|
||||
<h3>{{ status.targets.length > 1 ? target.name : t('ng_backup', 'Removed restore points') }}</h3>
|
||||
<ul>
|
||||
<li v-for="item in trash[target.id]" :key="item.id">
|
||||
{{ item.time ? formatDateTime(item.time) : item.id }}{{ item.label ? ' – ' + item.label : '' }}
|
||||
<span class="ngb-muted">{{ t('ng_backup', 'removed by {by}, deleted after {date}', { by: item.by, date: formatDate(item.deleteAfter) }) }}</span>
|
||||
<button v-if="item.by === RETENTION"
|
||||
class="ngb-info"
|
||||
:aria-expanded="retentionInfo ? 'true' : 'false'"
|
||||
:aria-label="t('ng_backup', 'What does this mean?')"
|
||||
:title="t('ng_backup', 'What does this mean?')"
|
||||
@click="retentionInfo = !retentionInfo">
|
||||
ⓘ
|
||||
</button>
|
||||
<NcButton variant="tertiary" @click="untrash(target, item)">{{ t('ng_backup', 'Restore') }}</NcButton>
|
||||
</li>
|
||||
</ul>
|
||||
<NcNoteCard v-if="retentionInfo && trash[target.id].some(item => item.by === RETENTION)" type="info">
|
||||
{{ n('ng_backup',
|
||||
'The retention policy keeps the newest restore points, and for older days, weeks and months only the last restore point of each. It moves the others to this bin, where they stay for %n day before they are deleted permanently. Until then you can bring one back with "Restore". Nothing was lost by accident.',
|
||||
'The retention policy keeps the newest restore points, and for older days, weeks and months only the last restore point of each. It moves the others to this bin, where they stay for %n days before they are deleted permanently. Until then you can bring one back with "Restore". Nothing was lost by accident.',
|
||||
status.key.deleteDelayDays) }}
|
||||
</NcNoteCard>
|
||||
</div>
|
||||
</template>
|
||||
</NcSettingsSection>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onBeforeUnmount, reactive, ref, watch } from 'vue'
|
||||
import { n, t } from '@nextcloud/l10n'
|
||||
import { confirmPassword } from '@nextcloud/password-confirmation'
|
||||
import NcButton from '@nextcloud/vue/components/NcButton'
|
||||
import NcLoadingIcon from '@nextcloud/vue/components/NcLoadingIcon'
|
||||
import NcNoteCard from '@nextcloud/vue/components/NcNoteCard'
|
||||
import NcProgressBar from '@nextcloud/vue/components/NcProgressBar'
|
||||
import NcSettingsSection from '@nextcloud/vue/components/NcSettingsSection'
|
||||
import api from '../api.js'
|
||||
import { formatDate, formatDateTime } from '../format.js'
|
||||
|
||||
const props = defineProps({ status: { type: Object, required: true } })
|
||||
const emit = defineEmits(['changed'])
|
||||
|
||||
// "by" of a restore point the weekly cleanup removed (PruneService)
|
||||
const RETENTION = 'retention policy'
|
||||
const retentionInfo = ref(false)
|
||||
const trash = reactive({})
|
||||
const reading = ref({ name: '', total: null, done: 0 }) // the location being read; null when done
|
||||
const error = ref('')
|
||||
const empty = computed(() => props.status.targets.every(target => !trash[target.id]?.length))
|
||||
|
||||
let current = 0 // the newest load; an older one stops (another took over, or the tab was left)
|
||||
onBeforeUnmount(() => { current++ })
|
||||
|
||||
async function loadTarget(target, load) {
|
||||
reading.value = { name: target.name, total: null, done: 0 }
|
||||
const items = []
|
||||
for (;;) {
|
||||
const page = await api.trash(target.id, items.length ? items[items.length - 1].id : '')
|
||||
if (load !== current) {
|
||||
return
|
||||
}
|
||||
items.push(...page.items)
|
||||
reading.value = { name: target.name, total: page.total, done: Math.min(items.length, page.total) }
|
||||
if (!page.items.length || items.length >= page.total) {
|
||||
break
|
||||
}
|
||||
}
|
||||
trash[target.id] = items.sort((a, b) => String(b.time ?? '').localeCompare(String(a.time ?? '')))
|
||||
}
|
||||
|
||||
async function loadTrash() {
|
||||
const load = ++current
|
||||
const failed = []
|
||||
for (const target of props.status.targets) {
|
||||
try {
|
||||
await loadTarget(target, load)
|
||||
} catch (e) {
|
||||
// keep what was shown: a location that cannot be reached does not mean its bin is empty
|
||||
failed.push(target.name + (e?.message ? ': ' + e.message : ''))
|
||||
}
|
||||
if (load !== current) {
|
||||
return
|
||||
}
|
||||
}
|
||||
error.value = failed.length ? t('ng_backup', 'The bin could not be read from {locations}', { locations: failed.join('; ') }) : ''
|
||||
reading.value = null
|
||||
}
|
||||
// Again when a run ends: a cleanup moves restore points here.
|
||||
watch(() => props.status.targets.map(tg => tg.id).join(',') + '|' + (props.status.runs ?? []).filter(run => run.status !== 'running').map(run => run.id).join(','), loadTrash, { immediate: true })
|
||||
|
||||
async function untrash(target, item) {
|
||||
error.value = ''
|
||||
try {
|
||||
await confirmPassword()
|
||||
await api.untrash(target.id, item.id)
|
||||
await loadTrash()
|
||||
emit('changed')
|
||||
} catch (e) {
|
||||
error.value = e?.message || ''
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.ngb-info { background: none; border: none; padding: 0 4px; min-height: auto; cursor: pointer; color: var(--color-primary-element); }
|
||||
.ngb-muted { color: var(--color-text-maxcontrast); }
|
||||
.ngb-reading { margin: 12px 0; display: flex; flex-direction: column; gap: 6px; max-width: 600px; }
|
||||
.ngb-reading p { margin: 0; }
|
||||
.ngb-trash { margin: 12px 0; }
|
||||
.ngb-trash h3 { font-weight: bold; }
|
||||
</style>
|
||||
@@ -4,15 +4,18 @@
|
||||
-->
|
||||
<template>
|
||||
<NcSettingsSection :name="t('ng_backup', 'Backup key and recovery kit')"
|
||||
:description="t('ng_backup', 'All backups are encrypted with one key. The server keeps it for scheduled backups. To restore on a new server you need the recovery kit and one of the passphrases.')">
|
||||
:description="t('ng_backup', 'All backups are encrypted with one key. The server keeps it for scheduled backups. To restore on a new server you need one of the passphrases; the recovery kit also helps when the key on the backup location is damaged.')">
|
||||
<!-- 1. Create the key -->
|
||||
<NcNoteCard v-if="!status.key.initialized" type="warning">
|
||||
{{ t('ng_backup', 'Restoring a server after a crash? Do not create a key here: a new key cannot read your backups. Follow the disaster recovery guide instead (occ backup:target:add asks for one of your passphrases, or import the recovery kit with occ backup:key:import).') }}
|
||||
</NcNoteCard>
|
||||
<form v-if="!status.key.initialized" class="ngb-form" @submit.prevent="createKey">
|
||||
<NcTextField v-model="label" :label="t('ng_backup', 'Who keeps this passphrase (e.g. Safe, CTO)')" />
|
||||
<NcPasswordField v-model="pass1" :label="t('ng_backup', 'Passphrase (at least 16 characters)')" autocomplete="new-password" />
|
||||
<NcPasswordField v-model="pass2" :label="t('ng_backup', 'Repeat passphrase')" autocomplete="new-password"
|
||||
:error="pass2 !== '' && pass1 !== pass2" :helper-text="pass2 !== '' && pass1 !== pass2 ? t('ng_backup', 'The passphrases do not match') : ''" />
|
||||
<NcTextField v-model="delay" type="number" min="1" max="365"
|
||||
:label="t('ng_backup', 'Days a removed backup stays in the trash before it is really deleted')"
|
||||
:label="t('ng_backup', 'Days a removed backup stays in the Bin before it is really deleted')"
|
||||
:helper-text="t('ng_backup', 'Protects against a malicious or mistaken cleanup. Cannot be changed later.')" />
|
||||
<NcButton type="submit" variant="primary" :disabled="busy || pass1.length < 16 || pass1 !== pass2 || !(delay >= 1 && delay <= 365)">
|
||||
{{ t('ng_backup', 'Create backup key') }}
|
||||
@@ -22,7 +25,7 @@
|
||||
<template v-else>
|
||||
<p>
|
||||
{{ t('ng_backup', 'Key fingerprint: {fingerprint}', { fingerprint: status.key.fingerprint }) }} ·
|
||||
{{ t('ng_backup', 'Removed backups are kept in the trash for {days} days', { days: status.key.deleteDelayDays }) }}
|
||||
{{ t('ng_backup', 'Removed backups are kept in the Bin for {days} days', { days: status.key.deleteDelayDays }) }}
|
||||
</p>
|
||||
|
||||
<!-- Passphrase slots -->
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
<!--
|
||||
- SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
- SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
-->
|
||||
<template>
|
||||
<div class="ngb-row">
|
||||
<strong class="ngb-name">{{ target.name }}</strong>
|
||||
<NcSelect v-model="kind" :options="kinds" label="label" :input-label="t('ng_backup', 'Schedule')" :clearable="false" />
|
||||
<NcTextField v-if="kind.id !== 'off'" v-model="time" class="ngb-time" :label="kind.id === 'daily' ? t('ng_backup', 'At (HH:MM, {zone})', { zone }) : t('ng_backup', 'From (HH:MM, {zone})', { zone })" />
|
||||
<NcTextField v-if="kind.id === 'hours'" v-model="hours" class="ngb-hours" type="number" min="1" max="23" :label="t('ng_backup', 'Every … hours')" />
|
||||
<NcButton :disabled="busy || wanted === current" @click="save">{{ t('ng_backup', 'Save') }}</NcButton>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import { t } from '@nextcloud/l10n'
|
||||
import NcButton from '@nextcloud/vue/components/NcButton'
|
||||
import NcSelect from '@nextcloud/vue/components/NcSelect'
|
||||
import NcTextField from '@nextcloud/vue/components/NcTextField'
|
||||
import api from '../api.js'
|
||||
|
||||
const props = defineProps({ target: { type: Object, required: true } })
|
||||
const emit = defineEmits(['changed', 'error'])
|
||||
|
||||
const kinds = [
|
||||
{ id: 'off', label: t('ng_backup', 'Off') },
|
||||
{ id: 'daily', label: t('ng_backup', 'Once a day') },
|
||||
{ id: 'hours', label: t('ng_backup', 'Every few hours') },
|
||||
]
|
||||
// A schedule is shown and edited in the zone it was saved in, named next to the time: "every 6
|
||||
// hours from 02:00" runs until midnight of that zone, so moving it to another zone would change
|
||||
// how often it runs. A location without a schedule gets the administrator's own zone (the browser's).
|
||||
const browserZone = Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC'
|
||||
const zone = computed(() => props.target.schedule !== '' && props.target.scheduleZone ? props.target.scheduleZone : browserZone)
|
||||
const kind = ref(kinds[0])
|
||||
const time = ref('02:00')
|
||||
const hours = ref('6')
|
||||
const busy = ref(false)
|
||||
|
||||
const current = computed(() => {
|
||||
const [at, every] = props.target.schedule.split('/')
|
||||
return at === '' ? '' : at + (every ? '/' + every : '')
|
||||
})
|
||||
const wanted = computed(() => {
|
||||
if (kind.value.id === 'off') {
|
||||
return ''
|
||||
}
|
||||
return time.value.trim() + (kind.value.id === 'hours' ? '/' + String(hours.value).trim() : '')
|
||||
})
|
||||
|
||||
watch(current, (schedule) => {
|
||||
const [at, every] = schedule.split('/')
|
||||
kind.value = kinds[at === '' ? 0 : (every ? 2 : 1)]
|
||||
time.value = at === '' ? time.value : at
|
||||
hours.value = every ?? hours.value
|
||||
}, { immediate: true })
|
||||
|
||||
async function save() {
|
||||
busy.value = true
|
||||
emit('error', '')
|
||||
try {
|
||||
await api.setTargetSchedule(props.target.id, wanted.value, zone.value)
|
||||
emit('changed')
|
||||
} catch (e) {
|
||||
emit('error', e.message)
|
||||
} finally {
|
||||
busy.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.ngb-row { display: flex; gap: 8px; align-items: flex-end; flex-wrap: wrap; margin-bottom: 12px; }
|
||||
.ngb-name { min-width: 120px; align-self: center; }
|
||||
.ngb-time { max-width: 220px; }
|
||||
.ngb-hours { max-width: 120px; }
|
||||
</style>
|
||||
@@ -11,6 +11,13 @@
|
||||
</div>
|
||||
|
||||
<template v-if="snapshot">
|
||||
<div class="ngb-row ngb-marks">
|
||||
<NcTextField v-model="note" class="ngb-note" :maxlength="500" :label="t('ng_backup', 'Note (can be changed later)')" />
|
||||
<NcButton :disabled="marking || note.trim() === (snapshot.note ?? '')" @click="saveNote">{{ t('ng_backup', 'Save note') }}</NcButton>
|
||||
<NcCheckboxRadioSwitch :model-value="snapshot.pinned" :disabled="marking" @update:model-value="savePinned">
|
||||
{{ t('ng_backup', 'Pin: the retention policy does not remove this restore point') }}
|
||||
</NcCheckboxRadioSwitch>
|
||||
</div>
|
||||
<div class="ngb-crumbs">
|
||||
<NcButton variant="tertiary" @click="open('data')">data</NcButton>
|
||||
<template v-for="(crumb, i) in crumbs" :key="crumb.path">
|
||||
@@ -49,6 +56,7 @@ import NcLoadingIcon from '@nextcloud/vue/components/NcLoadingIcon'
|
||||
import NcNoteCard from '@nextcloud/vue/components/NcNoteCard'
|
||||
import NcSelect from '@nextcloud/vue/components/NcSelect'
|
||||
import NcSettingsSection from '@nextcloud/vue/components/NcSettingsSection'
|
||||
import NcTextField from '@nextcloud/vue/components/NcTextField'
|
||||
import { confirmPassword } from '@nextcloud/password-confirmation'
|
||||
import '@nextcloud/password-confirmation/style.css'
|
||||
import api from '../api.js'
|
||||
@@ -74,7 +82,10 @@ const modes = [
|
||||
{ id: 'replace', label: t('ng_backup', 'Into the original place, overwrite and move files that are not in the backup to the trash') },
|
||||
]
|
||||
|
||||
const snapshotLabel = (s) => formatDateTime(s.createdAt) + (s.label ? ' – ' + s.label : '')
|
||||
const note = ref('')
|
||||
const marking = ref(false)
|
||||
|
||||
const snapshotLabel = (s) => (s.pinned ? '📌 ' : '') + formatDateTime(s.createdAt) + (s.label ? ' – ' + s.label : '') + (s.note ? ' – ' + s.note : '')
|
||||
const join = (name) => path.value + '/' + name
|
||||
const restorable = (p) => /^data\/[^/]+\/files(\/.*)?$/.test(p)
|
||||
const crumbs = computed(() => {
|
||||
@@ -88,7 +99,29 @@ watch(target, async (tg) => {
|
||||
snapshots.value = list.map(s => ({ ...s, displayLabel: snapshotLabel(s) }))
|
||||
snapshot.value = snapshots.value[0] ?? null
|
||||
}, { immediate: true })
|
||||
watch(snapshot, () => { selected.value = ''; open('data') })
|
||||
watch(snapshot, (s) => { selected.value = ''; note.value = s?.note ?? ''; open('data') })
|
||||
|
||||
// Changes the snapshot in place: choosing another one, which opens its folders again, is not needed
|
||||
async function mark(call, change) {
|
||||
const s = snapshot.value
|
||||
marking.value = true
|
||||
error.value = message.value = ''
|
||||
try {
|
||||
await call(target.value.id, s.id)
|
||||
change(s)
|
||||
s.displayLabel = snapshotLabel(s)
|
||||
emit('changed')
|
||||
} catch (e) {
|
||||
error.value = e.message
|
||||
} finally {
|
||||
marking.value = false
|
||||
}
|
||||
}
|
||||
const saveNote = () => {
|
||||
const text = note.value.trim()
|
||||
return mark((targetId, id) => api.setSnapshotNote(targetId, id, text), (s) => { s.note = text })
|
||||
}
|
||||
const savePinned = (pinned) => mark((targetId, id) => api.setSnapshotPinned(targetId, id, pinned), (s) => { s.pinned = pinned })
|
||||
|
||||
async function open(p) {
|
||||
if (!snapshot.value) {
|
||||
@@ -125,6 +158,8 @@ async function restore() {
|
||||
<style scoped>
|
||||
.ngb-row { display: flex; gap: 12px; flex-wrap: wrap; margin-bottom: 12px; }
|
||||
.ngb-row :deep(.ngb-snapshot) { min-width: min(360px, 100%); }
|
||||
.ngb-marks { align-items: flex-end; }
|
||||
.ngb-note { max-width: 360px; }
|
||||
.ngb-crumbs { display: flex; align-items: center; flex-wrap: wrap; }
|
||||
.ngb-entries li { display: flex; align-items: center; gap: 12px; min-height: 36px; }
|
||||
.ngb-entries li.selected { background: var(--color-primary-element-light); border-radius: var(--border-radius); }
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Backend;
|
||||
|
||||
use OCA\NgBackup\Backend\ExternalStorageFactory;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/** ExternalStorageFactory::normalizePrivateKey(): keys that phpseclib 2 read, for phpseclib 3 (#24). */
|
||||
class ExternalStorageFactoryTest extends TestCase {
|
||||
/** An OpenSSH-format key laid out as ssh-keygen writes it; the body is random, not a real key. */
|
||||
private static function key(): string {
|
||||
$body = base64_encode(random_bytes(1000));
|
||||
return "-----BEGIN OPENSSH PRIVATE KEY-----\n" . chunk_split($body, 70, "\n") . "-----END OPENSSH PRIVATE KEY-----\n";
|
||||
}
|
||||
|
||||
public function testRestoresLostLineBreaks(): void {
|
||||
$key = self::key();
|
||||
$this->assertSame($key, ExternalStorageFactory::normalizePrivateKey($key));
|
||||
$this->assertSame($key, ExternalStorageFactory::normalizePrivateKey(str_replace("\n", ' ', $key)), 'newlines as spaces');
|
||||
$this->assertSame($key, ExternalStorageFactory::normalizePrivateKey(str_replace("\n", '', $key)), 'no newlines');
|
||||
$this->assertSame($key, ExternalStorageFactory::normalizePrivateKey(str_replace("\n", "\r\n", $key)), 'CRLF');
|
||||
$this->assertSame($key, ExternalStorageFactory::normalizePrivateKey(rtrim($key)), 'no trailing newline');
|
||||
$this->assertSame($key, ExternalStorageFactory::normalizePrivateKey(preg_replace('/^/m', ' ', $key)), 'indented');
|
||||
}
|
||||
|
||||
public function testLeavesOtherFormatsAlone(): void {
|
||||
$pem = "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: DES-EDE3-CBC,FF44E0A94036766C\n\nAAAA\n-----END RSA PRIVATE KEY-----\n";
|
||||
$this->assertSame($pem, ExternalStorageFactory::normalizePrivateKey($pem));
|
||||
$this->assertSame('not a key', ExternalStorageFactory::normalizePrivateKey('not a key'));
|
||||
$odd = "-----BEGIN OPENSSH PRIVATE KEY-----\\nAAAA\\n-----END OPENSSH PRIVATE KEY-----";
|
||||
$this->assertSame($odd, ExternalStorageFactory::normalizePrivateKey($odd), 'not base64 inside: unchanged');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Backend;
|
||||
|
||||
use OCA\NgBackup\Backend\SftpRangeFetcher;
|
||||
use OCA\NgBackup\Backend\SftpReadStream;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
require_once __DIR__ . '/SftpReadStreamTest.php'; // FakeSftpConnection, FakeSftpStorage
|
||||
|
||||
/** SftpRangeFetcher against the fake SFTP server of SftpReadStreamTest, several connections at once. */
|
||||
class SftpRangeFetcherTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
/** @var array<string, string> */
|
||||
private array $files = [];
|
||||
|
||||
protected function setUp(): void {
|
||||
SftpReadStream::reset();
|
||||
for ($i = 0; $i < 10; $i++) {
|
||||
$this->files["/home/repo/packs/p$i"] = random_bytes(3 * 1048576 + $i * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
/** @return list<FakeSftpConnection> */
|
||||
private function connections(int $n, array $options = []): array {
|
||||
$result = [];
|
||||
for ($c = 0; $c < $n; $c++) {
|
||||
$result[] = new FakeSftpConnection($this->files, ...($options[$c] ?? []));
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
/** @return list<object> what StorageBackend passes on: each with its SFTP channel open */
|
||||
private static function opened(array $connections): array {
|
||||
return array_map(fn (FakeSftpConnection $c): object => SftpReadStream::connection(new FakeSftpStorage($c)), $connections);
|
||||
}
|
||||
|
||||
/** @return list<array{0:string, 1:int, 2:int}> */
|
||||
private function ranges(): array {
|
||||
$ranges = [];
|
||||
for ($i = 0; $i < 10; $i++) {
|
||||
$ranges[] = ["repo/packs/p$i", 1000 * $i, 2 * 1048576 + 333];
|
||||
}
|
||||
return $ranges;
|
||||
}
|
||||
|
||||
/** @return array<int, string|null> sha1 of the contents by position (a failing comparison of MiBs would not fit in memory) */
|
||||
private static function collect(\Generator $streams): array {
|
||||
$result = [];
|
||||
foreach ($streams as $i => $fh) {
|
||||
$result[$i] = $fh === null ? null : sha1(stream_get_contents($fh));
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function expected(array $ranges): array {
|
||||
return array_map(fn (array $r): string => sha1(substr($this->files['/home/' . $r[0]], $r[1], $r[2])), $ranges);
|
||||
}
|
||||
|
||||
public function testReadsEveryRangeInOrderOverAllConnections(): void {
|
||||
$connections = $this->connections(4);
|
||||
$ranges = $this->ranges();
|
||||
$spool = $this->tempDir();
|
||||
|
||||
$this->assertSame($this->expected($ranges), self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', $ranges, $spool)));
|
||||
|
||||
foreach ($connections as $c) {
|
||||
$this->assertGreaterThan(0, count($c->reads), 'every connection reads');
|
||||
$this->assertLessThanOrEqual(SftpRangeFetcher::DEPTH, $c->maxInFlight);
|
||||
$this->assertSame([], $c->openHandles);
|
||||
$this->assertSame(0, $c->pending());
|
||||
$this->assertTrue($c->disconnected, 'its own connections end with it');
|
||||
}
|
||||
$this->assertSame([], glob("$spool/*"), 'nothing left in the spool');
|
||||
}
|
||||
|
||||
public function testKeepsSeveralRequestsInFlightOnEachConnection(): void {
|
||||
$this->files['/home/repo/packs/big'] = random_bytes(10 * 1048576);
|
||||
$connections = $this->connections(2);
|
||||
self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', [['repo/packs/big', 0, 10 * 1048576], ['repo/packs/big', 5, 10 * 1048576 - 5]], $this->tempDir()));
|
||||
|
||||
$this->assertSame(SftpRangeFetcher::DEPTH, $connections[0]->maxInFlight);
|
||||
$this->assertSame(SftpRangeFetcher::DEPTH, $connections[1]->maxInFlight);
|
||||
}
|
||||
|
||||
public function testAnAnswerPhpseclibHasReadAlreadyIsTakenWithoutWaitingOnTheSocket(): void {
|
||||
// The fake's answers are all "read already": its socket never becomes readable, and
|
||||
// phpseclib's packet buffer holds data. Waiting on the socket would take a second a step.
|
||||
$connections = $this->connections(2);
|
||||
$sockets = [];
|
||||
foreach ($connections as $c) {
|
||||
$pair = stream_socket_pair(STREAM_PF_UNIX, STREAM_SOCK_STREAM, STREAM_IPPROTO_IP);
|
||||
$sockets[] = $pair;
|
||||
$c->fsock = $pair[0];
|
||||
$c->packet_buffer = pack('N', 4) . 'more'; // a whole packet
|
||||
}
|
||||
$ranges = array_slice($this->ranges(), 0, 3);
|
||||
$start = microtime(true);
|
||||
|
||||
$this->assertSame($this->expected($ranges), self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', $ranges, $this->tempDir())));
|
||||
|
||||
$this->assertLessThan(1.0, microtime(true) - $start);
|
||||
}
|
||||
|
||||
public function testAServerThatCapsAReadStillGivesEveryByte(): void {
|
||||
$connections = $this->connections(3, array_fill(0, 3, ['cap' => 100000]));
|
||||
$ranges = $this->ranges();
|
||||
|
||||
$this->assertSame($this->expected($ranges), self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', $ranges, $this->tempDir())));
|
||||
// learned: after the first short answers (and asking for their rest) it asks for no more than the server gives
|
||||
foreach ($connections as $c) {
|
||||
$this->assertLessThanOrEqual(2 * SftpRangeFetcher::DEPTH, count(array_filter($c->reads, fn (int $n): bool => $n > 100000)));
|
||||
}
|
||||
}
|
||||
|
||||
public function testAMissingFileIsLeftToTheCaller(): void {
|
||||
$connections = $this->connections(2);
|
||||
$ranges = $this->ranges();
|
||||
$ranges[3][0] = 'repo/packs/gone';
|
||||
|
||||
$result = self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', $ranges, $this->tempDir()));
|
||||
|
||||
$expected = $this->expected($this->ranges());
|
||||
$expected[3] = null;
|
||||
$this->assertSame($expected, $result);
|
||||
$this->assertSame([], $connections[0]->openHandles);
|
||||
}
|
||||
|
||||
public function testARangePastTheEndOfTheFileIsLeftToTheCaller(): void {
|
||||
$ranges = $this->ranges();
|
||||
$ranges[5][2] = 10 * 1048576;
|
||||
|
||||
$result = self::collect(SftpRangeFetcher::fetch(self::opened($this->connections(2)), '/home/', $ranges, $this->tempDir()));
|
||||
|
||||
$this->assertNull($result[5]);
|
||||
$this->assertSame($this->expected($this->ranges())[6], $result[6]);
|
||||
}
|
||||
|
||||
public function testAServerErrorFailsOnlyThatRange(): void {
|
||||
// every read from 1 MiB on fails on connection 0: its ranges, not those of the others
|
||||
$connections = $this->connections(2, [['failAt' => 1048576]]);
|
||||
$ranges = $this->ranges();
|
||||
|
||||
$result = self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', $ranges, $this->tempDir()));
|
||||
|
||||
$expected = $this->expected($ranges);
|
||||
$failed = array_keys(array_filter($result, fn (?string $r): bool => $r === null));
|
||||
$this->assertNotSame([], $failed);
|
||||
foreach ($result as $i => $data) {
|
||||
if ($data !== null) {
|
||||
$this->assertSame($expected[$i], $data);
|
||||
}
|
||||
}
|
||||
$this->assertLessThan(10, count($failed), 'the other connection goes on');
|
||||
$this->assertSame([], $connections[0]->openHandles, 'a failed range is closed too');
|
||||
}
|
||||
|
||||
public function testALostConnectionIsNotUsedAgainAndTheOthersGoOn(): void {
|
||||
$connections = $this->connections(3, [['loseAt' => 1048576]]);
|
||||
$ranges = $this->ranges();
|
||||
|
||||
$result = self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', $ranges, $this->tempDir()));
|
||||
|
||||
$expected = $this->expected($ranges);
|
||||
$this->assertSame(1, count(array_filter($result, fn (?string $r): bool => $r === null)), 'only the range it was reading');
|
||||
foreach ($result as $i => $data) {
|
||||
if ($data !== null) {
|
||||
$this->assertSame($expected[$i], $data);
|
||||
}
|
||||
}
|
||||
$this->assertTrue($connections[0]->disconnected);
|
||||
$this->assertLessThanOrEqual(1048576 / 261120 + SftpRangeFetcher::DEPTH + 1, count($connections[0]->reads), 'not used after it broke');
|
||||
}
|
||||
|
||||
public function testWhenEveryConnectionIsLostTheRestIsLeftToTheCaller(): void {
|
||||
$connections = $this->connections(2, [['throwAt' => 1048576], ['loseAt' => 1048576]]);
|
||||
|
||||
$result = self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', $this->ranges(), $this->tempDir()));
|
||||
|
||||
$this->assertSame(array_fill(0, 10, null), $result);
|
||||
}
|
||||
|
||||
public function testABrokenConnectionIsReplacedAndTheRestIsReadAhead(): void {
|
||||
$connections = $this->connections(1, [['loseAt' => 1048576]]);
|
||||
$made = [];
|
||||
$connect = function () use (&$made): object {
|
||||
$made[] = $c = new FakeSftpConnection($this->files);
|
||||
return SftpReadStream::connection(new FakeSftpStorage($c));
|
||||
};
|
||||
$ranges = $this->ranges();
|
||||
|
||||
$result = self::collect(SftpRangeFetcher::fetch(self::opened($connections), '/home/', $ranges, $this->tempDir(), $connect));
|
||||
|
||||
$expected = $this->expected($ranges);
|
||||
$expected[0] = null; // the range it was reading when it broke: read the usual way
|
||||
$this->assertSame($expected, $result);
|
||||
$this->assertCount(1, $made);
|
||||
$this->assertTrue($made[0]->disconnected, 'the new one ends with the fetcher too');
|
||||
}
|
||||
|
||||
public function testConnectionsThatKeepBreakingAreReplacedOnlySoOften(): void {
|
||||
$made = 0;
|
||||
$connect = function () use (&$made): object {
|
||||
$made++;
|
||||
return SftpReadStream::connection(new FakeSftpStorage(new FakeSftpConnection($this->files, loseAt: 0)));
|
||||
};
|
||||
$ranges = array_merge($this->ranges(), $this->ranges(), $this->ranges());
|
||||
|
||||
$result = self::collect(SftpRangeFetcher::fetch(self::opened($this->connections(1, [['loseAt' => 0]])), '/home/', $ranges, $this->tempDir(), $connect));
|
||||
|
||||
$this->assertSame(array_fill(0, count($ranges), null), $result);
|
||||
$this->assertSame(SftpRangeFetcher::MAX_RECONNECTS, $made);
|
||||
}
|
||||
|
||||
public function testStoppingHalfwayEndsTheConnectionsAndLeavesNothingBehind(): void {
|
||||
$connections = $this->connections(2);
|
||||
$spool = $this->tempDir();
|
||||
$streams = SftpRangeFetcher::fetch(self::opened($connections), '/home/', $this->ranges(), $spool);
|
||||
$this->assertIsResource($streams->current());
|
||||
|
||||
unset($streams);
|
||||
|
||||
$this->assertTrue($connections[0]->disconnected);
|
||||
$this->assertTrue($connections[1]->disconnected);
|
||||
$this->assertSame([], glob("$spool/*"));
|
||||
}
|
||||
|
||||
public function testReadsNoMoreThanAheadOfTheCaller(): void {
|
||||
$connections = $this->connections(4);
|
||||
$streams = SftpRangeFetcher::fetch(self::opened($connections), '/home/', $this->ranges(), $this->tempDir());
|
||||
$streams->current();
|
||||
|
||||
$opened = array_sum(array_map(fn (FakeSftpConnection $c): int => count(array_unique(array_values($c->opened))), $connections));
|
||||
$this->assertLessThanOrEqual(SftpRangeFetcher::AHEAD, $opened);
|
||||
}
|
||||
}
|
||||
@@ -466,6 +466,12 @@ final class FakeSftpConnection {
|
||||
public array $reads = [];
|
||||
/** @var array<string, string> */
|
||||
public array $openHandles = [];
|
||||
/** @var list<string> paths of every OPEN request */
|
||||
public array $opened = [];
|
||||
/** @var resource|null like phpseclib's socket (SftpRangeFetcher waits on it) */
|
||||
public $fsock = null;
|
||||
/** like phpseclib's: what came in with the last packet and is not read yet */
|
||||
public string $packet_buffer = '';
|
||||
/** Whether the SFTP channel is open: phpseclib 2 opens it on the first SFTP call. */
|
||||
public bool $initialised = false;
|
||||
/** Packets sent */
|
||||
@@ -532,6 +538,7 @@ final class FakeSftpConnection {
|
||||
return false;
|
||||
}
|
||||
$path = substr($data, 4, unpack('N', $data)[1]);
|
||||
$this->opened[] = $path;
|
||||
if (!isset($this->files[$path])) {
|
||||
$this->answers[$id] = [101, pack('N', 2)];
|
||||
break;
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Backend;
|
||||
|
||||
use OCA\NgBackup\Backend\SftpRangeFetcher;
|
||||
use OCA\NgBackup\Backend\SftpReadStream;
|
||||
use OCA\NgBackup\Backend\StorageBackend;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use OCP\Files\Storage\IStorage;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
require_once __DIR__ . '/SftpReadStreamTest.php'; // FakeSftpConnection
|
||||
require_once __DIR__ . '/StorageBackendReconnectTest.php'; // SftpLikeStorage
|
||||
|
||||
/** StorageBackend::readAhead(): over SFTP on connections of its own, built with the reconnect function. */
|
||||
class StorageBackendReadAheadTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
/** @var list<FakeSftpConnection> */
|
||||
private array $connections = [];
|
||||
/** @var array<string, string> */
|
||||
private array $files = [];
|
||||
/** New storages after this many fail to connect. */
|
||||
private ?int $connectLimit = null;
|
||||
|
||||
protected function setUp(): void {
|
||||
SftpReadStream::reset();
|
||||
for ($i = 0; $i < 6; $i++) {
|
||||
$this->files["/home/base/repo/packs/p$i"] = random_bytes(1048576 + $i);
|
||||
}
|
||||
}
|
||||
|
||||
private function storage(): IStorage {
|
||||
if ($this->connectLimit !== null && count($this->connections) >= $this->connectLimit) {
|
||||
throw new \RuntimeException('Too many connections');
|
||||
}
|
||||
$sftp = new FakeSftpConnection($this->files);
|
||||
$this->connections[] = $sftp;
|
||||
$storage = $this->createMock(SftpLikeRootStorage::class);
|
||||
$storage->method('instanceOfStorage')->willReturnCallback(fn (string $class): bool => $class === \OCA\Files_External\Lib\Storage\SFTP::class);
|
||||
$storage->method('getConnection')->willReturn($sftp);
|
||||
$storage->method('getRoot')->willReturn('/home/');
|
||||
$storage->method('is_dir')->willReturn(true);
|
||||
return $storage;
|
||||
}
|
||||
|
||||
/** @return list<array{0:string, 1:int, 2:int}> */
|
||||
private static function ranges(): array {
|
||||
return array_map(fn (int $i): array => ["packs/p$i", 7, 1048576 - 7], range(0, 5));
|
||||
}
|
||||
|
||||
public function testReadsAheadOnNewConnectionsOnly(): void {
|
||||
$backend = new StorageBackend($this->storage(), 'base/repo', false, fn (): IStorage => $this->storage());
|
||||
$backend->readAheadConnections = 4;
|
||||
$this->assertTrue($backend->canReadAhead());
|
||||
|
||||
$result = [];
|
||||
foreach ($backend->readAhead(self::ranges(), $this->tempDir()) as $i => $fh) {
|
||||
$result[$i] = sha1(stream_get_contents($fh));
|
||||
}
|
||||
|
||||
$expected = array_map(fn (int $i): string => sha1(substr($this->files["/home/base/repo/packs/p$i"], 7, 1048576 - 7)), range(0, 5));
|
||||
$this->assertSame($expected, $result);
|
||||
$this->assertCount(1 + 4, $this->connections);
|
||||
$this->assertSame(0, $this->connections[0]->sent, 'the storage\'s own connection stays free');
|
||||
foreach (array_slice($this->connections, 1) as $c) {
|
||||
$this->assertTrue($c->disconnected);
|
||||
}
|
||||
}
|
||||
|
||||
public function testGoesOnWithTheConnectionsTheServerAllows(): void {
|
||||
$this->connectLimit = 3;
|
||||
$backend = new StorageBackend($this->storage(), 'base/repo', false, fn (): IStorage => $this->storage());
|
||||
$backend->readAheadConnections = 4;
|
||||
|
||||
$count = 0;
|
||||
foreach ($backend->readAhead(self::ranges(), $this->tempDir()) as $fh) {
|
||||
$this->assertIsResource($fh);
|
||||
$count++;
|
||||
}
|
||||
|
||||
$this->assertSame(6, $count);
|
||||
$this->assertCount(3, $this->connections);
|
||||
}
|
||||
|
||||
public function testByDefaultOneConnectionNextToTheStoragesOwn(): void {
|
||||
$backend = new StorageBackend($this->storage(), 'base/repo', false, fn (): IStorage => $this->storage());
|
||||
foreach ($backend->readAhead(self::ranges(), $this->tempDir()) as $fh) {
|
||||
$this->assertIsResource($fh);
|
||||
}
|
||||
|
||||
$this->assertSame(1, SftpRangeFetcher::DEFAULT_CONNECTIONS);
|
||||
$this->assertCount(2, $this->connections);
|
||||
}
|
||||
|
||||
public function testNoNewConnectionMeansNoReadingAhead(): void {
|
||||
$this->connectLimit = 1;
|
||||
$backend = new StorageBackend($this->storage(), 'base/repo', false, fn (): IStorage => $this->storage());
|
||||
|
||||
$this->assertNull($backend->readAhead(self::ranges(), $this->tempDir()));
|
||||
}
|
||||
|
||||
public function testWithoutAWayToConnectAgainItDoesNotReadAhead(): void {
|
||||
$backend = new StorageBackend($this->storage(), 'base/repo', false, null);
|
||||
|
||||
$this->assertFalse($backend->canReadAhead());
|
||||
$this->assertNull($backend->readAhead(self::ranges(), $this->tempDir()));
|
||||
}
|
||||
}
|
||||
|
||||
interface SftpLikeRootStorage extends SftpLikeStorage {
|
||||
public function getRoot(): string;
|
||||
}
|
||||
@@ -77,6 +77,24 @@ class StorageBackendReconnectTest extends TestCase {
|
||||
$this->assertSame(1, $backend->reconnects);
|
||||
}
|
||||
|
||||
public function testASilentlyDroppedConnectionIsReplacedAfterAPause(): void {
|
||||
$backend = $this->backend();
|
||||
$this->connections[0]->dropped = true;
|
||||
$this->connections[0]->silent = true; // phpseclib still says connected
|
||||
self::idle($backend);
|
||||
$this->assertTrue($backend->exists('config'));
|
||||
$this->assertSame(1, $backend->reconnects, 'the check failed, so the connection was replaced');
|
||||
}
|
||||
|
||||
public function testAFailureOnASilentlyDroppedConnectionIsRepeated(): void {
|
||||
$backend = $this->backend();
|
||||
$this->connections[0]->dropped = true;
|
||||
$this->connections[0]->silent = true;
|
||||
$fh = $backend->get('config');
|
||||
$this->assertSame('config', stream_get_contents($fh));
|
||||
$this->assertSame(1, $backend->reconnects);
|
||||
}
|
||||
|
||||
public function testAMissingFileOnALiveConnectionIsJustMissing(): void {
|
||||
$backend = $this->backend();
|
||||
$this->assertFalse($backend->exists('nothing'));
|
||||
@@ -121,6 +139,8 @@ interface SftpLikeStorage extends IStorage {
|
||||
/** The public phpseclib calls the backend uses to tell a dead connection. */
|
||||
final class ReconnectFakeSftp {
|
||||
public bool $dropped = false;
|
||||
/** Dropped without phpseclib noticing (as a Hetzner Storage Box does): it still says connected. */
|
||||
public bool $silent = false;
|
||||
private bool $connected = true;
|
||||
public int $requests = 0;
|
||||
|
||||
@@ -139,6 +159,9 @@ final class ReconnectFakeSftp {
|
||||
public function request(): bool {
|
||||
$this->requests++;
|
||||
if ($this->dropped) {
|
||||
if ($this->silent) {
|
||||
return false;
|
||||
}
|
||||
$this->connected = false;
|
||||
}
|
||||
return $this->connected;
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Command;
|
||||
|
||||
use OCA\NgBackup\Command\TargetAdd;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class TargetAddEstimateTest extends TestCase {
|
||||
public function testNoEstimateFromTheFirstFewFiles(): void {
|
||||
// 4,568 index files, 10 in after 6 s: too few to say anything
|
||||
$this->assertNull(TargetAdd::minutesLeft(6.0, 10, 4558, 4568));
|
||||
}
|
||||
|
||||
public function testEstimateOnceEnoughFilesCameIn(): void {
|
||||
// 300 files in 10 s: 4,268 left at 30 a second is about 2.4 minutes
|
||||
$this->assertSame(3, TargetAdd::minutesLeft(10.0, 300, 4268, 4568));
|
||||
}
|
||||
|
||||
public function testFivePercentOfASmallIndexIsEnough(): void {
|
||||
// 6 of 100 files in 6 s: 94 left at one a second is about 1.6 minutes
|
||||
$this->assertSame(2, TargetAdd::minutesLeft(6.0, 6, 94, 100));
|
||||
}
|
||||
|
||||
public function testNoEstimateInTheFirstSecondsOrWhenDone(): void {
|
||||
$this->assertNull(TargetAdd::minutesLeft(2.0, 500, 4000, 4500));
|
||||
$this->assertNull(TargetAdd::minutesLeft(60.0, 4568, 0, 4568));
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,27 @@ class KeyRingTest extends TestCase {
|
||||
KeyRing::unwrap($wrapped, 'wrong');
|
||||
}
|
||||
|
||||
public function testInvalidBase64IsADamagedKey(): void {
|
||||
$wrapped = KeyRing::generate()->wrap('right');
|
||||
$wrapped['salt'] = '***not base64***';
|
||||
$this->expectException(CryptoException::class);
|
||||
$this->expectExceptionMessage('Damaged key file');
|
||||
KeyRing::unwrapAny($wrapped, 'right');
|
||||
}
|
||||
|
||||
public function testAShortSaltOrNonceIsADamagedKey(): void {
|
||||
foreach (['salt', 'nonce'] as $part) {
|
||||
$wrapped = KeyRing::generate()->wrap('right');
|
||||
$wrapped[$part] = base64_encode('short');
|
||||
try {
|
||||
KeyRing::unwrapAny($wrapped, 'right');
|
||||
$this->fail("a short $part was accepted");
|
||||
} catch (CryptoException $e) {
|
||||
$this->assertStringContainsString('Damaged key file', $e->getMessage(), $part);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function testBlobIdsAreKeyed(): void {
|
||||
$this->assertNotSame(KeyRing::generate()->blobId('same'), KeyRing::generate()->blobId('same'));
|
||||
$k = KeyRing::generate();
|
||||
@@ -34,6 +55,20 @@ class KeyRingTest extends TestCase {
|
||||
new KeyRing('short');
|
||||
}
|
||||
|
||||
public function testADamagedSlotDoesNotStopTheNextOne(): void {
|
||||
$keys = KeyRing::generate();
|
||||
$broken = $keys->wrap('passphrase in the safe');
|
||||
$broken['salt'] = '*** not base64 ***';
|
||||
$field = ['slots' => [
|
||||
['slot' => 1, 'label' => 'Safe', 'key' => $broken],
|
||||
['slot' => 2, 'label' => 'Missing parts', 'key' => ['v' => 1]],
|
||||
['slot' => 3, 'label' => 'CTO', 'key' => $keys->wrap('passphrase of the cto')],
|
||||
]];
|
||||
$this->assertSame($keys->blobId('x'), KeyRing::unwrapAny($field, 'passphrase of the cto')->blobId('x'));
|
||||
$this->expectException(CryptoException::class);
|
||||
KeyRing::unwrapAny($field, 'passphrase in the safe');
|
||||
}
|
||||
|
||||
public function testAnySlotOpensTheSameKey(): void {
|
||||
$keys = KeyRing::generate();
|
||||
$field = ['slots' => [
|
||||
|
||||
@@ -14,4 +14,11 @@ class FormatTest extends TestCase {
|
||||
$this->assertSame('1d 00:00:00', Format::duration(86400));
|
||||
$this->assertSame('1d 12:10:13', Format::duration(86400 + 12 * 3600 + 10 * 60 + 13));
|
||||
}
|
||||
|
||||
public function testTimeInZone(): void {
|
||||
$t = (int)strtotime('2026-10-19 13:12:00 UTC');
|
||||
$this->assertSame('2026-10-19 13:12', Format::time($t, 'UTC'));
|
||||
$this->assertSame('2026-10-19 15:12', Format::time($t, 'Europe/Berlin')); // summer time
|
||||
$this->assertSame('2026-12-01 01:00', Format::time((int)strtotime('2026-12-01 00:00:00 UTC'), 'Europe/Amsterdam'));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,4 +24,25 @@ class BackupProgressTest extends TestCase {
|
||||
$this->assertNull(BackupRun::progress(['phase' => 'files', 'estimate' => ['bytes' => 0, 'approximate' => true], 'stats' => ['bytes' => 1]]));
|
||||
$this->assertNull(BackupRun::progress(['phase' => 'commit', 'estimate' => ['bytes' => 10, 'approximate' => false], 'stats' => ['bytes' => 1]]));
|
||||
}
|
||||
|
||||
private static function runIn(string $phase, array $state, string $status = \OCA\NgBackup\Db\Run::RUNNING): \OCA\NgBackup\Db\Run {
|
||||
$run = new \OCA\NgBackup\Db\Run();
|
||||
$run->setStatus($status);
|
||||
$run->setPhase($phase);
|
||||
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
|
||||
return $run;
|
||||
}
|
||||
|
||||
public function testDatabasePhaseCountsTheTablesDumped(): void {
|
||||
$run = self::runIn('db', ['dump' => ['tables' => array_fill(0, 183, 'oc_t'), 'i' => 47]]);
|
||||
$this->assertSame(['fraction' => 47 / 183, 'approximate' => false, 'tables' => 47, 'totalTables' => 183], BackupRun::progressOf($run));
|
||||
}
|
||||
|
||||
public function testDatabasePhaseHasNoFigureBeforeItsFirstStep(): void {
|
||||
$this->assertNull(BackupRun::progressOf(self::runIn('db', ['files' => []])));
|
||||
}
|
||||
|
||||
public function testNoProgressForAFinishedRun(): void {
|
||||
$this->assertNull(BackupRun::progressOf(self::runIn('db', ['dump' => ['tables' => ['oc_a'], 'i' => 0]], \OCA\NgBackup\Db\Run::DONE)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,28 @@ class TreeWalkerTest extends TestCase {
|
||||
$this->assertSame(['Z', 'a', 'a-b', 'a.txt', 'b', 'skip'], array_values(array_unique(array_map(fn ($p) => explode('/', $p)[0], $paths))));
|
||||
}
|
||||
|
||||
public function testEmptyDirectoriesAreYieldedInOrderAndOnResume(): void {
|
||||
$d = $this->tree();
|
||||
mkdir("$d/a/empty");
|
||||
mkdir("$d/a-hollow/inner", 0700, true);
|
||||
symlink("$d/a.txt", "$d/skip/link");
|
||||
mkdir("$d/only-link");
|
||||
symlink("$d/a.txt", "$d/only-link/link");
|
||||
$w = new TreeWalker(['' => $d]);
|
||||
$all = array_keys(iterator_to_array($w->walk()));
|
||||
$this->assertContains('a/empty', $all);
|
||||
$this->assertContains('a-hollow/inner', $all);
|
||||
$this->assertNotContains('a-hollow', $all, 'only the deepest');
|
||||
$this->assertContains('only-link', $all, 'a link is not walked');
|
||||
$this->assertNotContains('skip', $all);
|
||||
$sorted = $all;
|
||||
usort($sorted, [TreeWalker::class, 'compare']);
|
||||
$this->assertSame($sorted, $all);
|
||||
foreach ($all as $i => $cursor) {
|
||||
$this->assertSame(array_slice($all, $i + 1), array_keys(iterator_to_array($w->walk($cursor))), "after $cursor");
|
||||
}
|
||||
}
|
||||
|
||||
public function testResumeAfterCursorYieldsTheRest(): void {
|
||||
$w = new TreeWalker(['' => $this->tree()]);
|
||||
$all = array_keys(iterator_to_array($w->walk()));
|
||||
|
||||
@@ -50,6 +50,40 @@ class CatalogTest extends TestCase {
|
||||
$this->assertSame(4, $again->catalogGeneration());
|
||||
}
|
||||
|
||||
public function testTheTrashIsReadInPartsThatTogetherAreAll(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
for ($i = 0; $i < 3; $i++) {
|
||||
$repo->forget($repo->backupDirectory($this->src)['snapshot']);
|
||||
}
|
||||
$again = $this->open();
|
||||
$seen = [];
|
||||
// A deadline already past: one entry per request, as on a slow location
|
||||
$after = '';
|
||||
for ($request = 0; count($seen) < 3 && $request < 5; $request++) {
|
||||
$page = $again->trashPage($after, 0.0);
|
||||
$this->assertSame(3, $page['total']);
|
||||
$this->assertCount(1, $page['items']);
|
||||
$seen += $page['items'];
|
||||
$after = (string)array_key_last($page['items']);
|
||||
}
|
||||
$this->assertEqualsCanonicalizing(array_keys($again->trash()), array_keys($seen));
|
||||
$this->assertSame(['total' => 3, 'items' => []], $again->trashPage($after, 0.0));
|
||||
$this->assertCount(3, $again->trashPage('', microtime(true) + 60)['items']);
|
||||
}
|
||||
|
||||
public function testAnEntryThatLeavesTheTrashBetweenPartsMakesNoneComeTwiceOrBeMissed(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
for ($i = 0; $i < 3; $i++) {
|
||||
$repo->forget($repo->backupDirectory($this->src)['snapshot']);
|
||||
}
|
||||
$reader = $this->open();
|
||||
$ids = array_keys($reader->trashPage('', microtime(true) + 60)['items']);
|
||||
$first = $reader->trashPage('', 0.0)['items'];
|
||||
$this->assertSame([$ids[0]], array_keys($first));
|
||||
$this->open()->untrash($ids[0]); // another administrator, between two parts
|
||||
$this->assertSame([$ids[1], $ids[2]], array_keys($this->open()->trashPage($ids[0], microtime(true) + 60)['items']));
|
||||
}
|
||||
|
||||
public function testRollbackIsDetected(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
$repo->backupDirectory($this->src);
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/**
|
||||
* A pack stream that hands out wrong bytes once (an SMB restore stopped after 17 hours with
|
||||
* "Decryption failed"): the blob is read once more on a request of its own; only data that is
|
||||
* damaged on the location stops the restore.
|
||||
*/
|
||||
class DamagedReadTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private string $root;
|
||||
private KeyRing $keys;
|
||||
private string $src;
|
||||
private string $snapshot;
|
||||
|
||||
protected function setUp(): void {
|
||||
$this->root = $this->tempDir();
|
||||
$this->keys = KeyRing::generate();
|
||||
$this->src = $this->tempDir();
|
||||
for ($i = 0; $i < 6; $i++) {
|
||||
file_put_contents("$this->src/f$i", random_bytes(3 * 1048576));
|
||||
}
|
||||
$repo = Repository::initWithKey(new LocalBackend($this->root), $this->keys, $this->keys->wrap('pw'));
|
||||
$this->snapshot = $repo->backupDirectory($this->src)['snapshot'];
|
||||
}
|
||||
|
||||
public function testABlobThatCameInDamagedOnceIsReadAgain(): void {
|
||||
$backend = new GarblingBackend(new LocalBackend($this->root), streams: 1);
|
||||
$target = $this->tempDir();
|
||||
|
||||
$this->assertSame(6, Repository::openWithKey($backend, $this->keys)->restore($this->snapshot, $target));
|
||||
|
||||
$this->assertSame(self::hashTree($this->src), self::hashTree($target));
|
||||
$this->assertSame(0, $backend->streams, 'the damaged read happened');
|
||||
}
|
||||
|
||||
public function testATableDumpThatCameInDamagedOnceIsReadAgain(): void {
|
||||
$repo = Repository::openWithKey(new LocalBackend($this->root), $this->keys);
|
||||
$stats = [];
|
||||
$writer = $repo->blobWriter($stats);
|
||||
$lines = [];
|
||||
for ($i = 0; $i < 20000; $i++) {
|
||||
$lines[] = json_encode(['i' => $i, 'pad' => str_repeat('x', $i % 400)]);
|
||||
}
|
||||
$writer->write(implode("\n", $lines) . "\n");
|
||||
$blobs = $writer->finish();
|
||||
$repo->flushPacks();
|
||||
$backend = new GarblingBackend(new LocalBackend($this->root), ranges: 1);
|
||||
|
||||
$this->assertSame($lines, iterator_to_array(Repository::openWithKey($backend, $this->keys)->readLines($blobs), false));
|
||||
$this->assertSame(0, $backend->ranges, 'the damaged read happened');
|
||||
}
|
||||
|
||||
public function testDataDamagedOnTheLocationStillStopsTheRestore(): void {
|
||||
$backend = new GarblingBackend(new LocalBackend($this->root), streams: 1000, ranges: 1000);
|
||||
|
||||
$this->expectException(\Exception::class);
|
||||
Repository::openWithKey($backend, $this->keys)->restore($this->snapshot, $this->tempDir());
|
||||
}
|
||||
}
|
||||
|
||||
/** Pack reads that flip a byte: the first $streams pack streams and $ranges range reads. */
|
||||
final class GarblingBackend implements IBackend {
|
||||
use \OCA\NgBackup\Backend\GetsOneByOne;
|
||||
|
||||
public function __construct(
|
||||
private IBackend $inner,
|
||||
public int $streams = 0,
|
||||
public int $ranges = 0,
|
||||
) {
|
||||
}
|
||||
|
||||
public function put(string $path, $stream): void {
|
||||
$this->inner->put($path, $stream);
|
||||
}
|
||||
|
||||
public function get(string $path) {
|
||||
$fh = $this->inner->get($path);
|
||||
if (!str_starts_with($path, 'packs/') || $this->streams <= 0) {
|
||||
return $fh;
|
||||
}
|
||||
$this->streams--;
|
||||
$data = stream_get_contents($fh);
|
||||
fclose($fh);
|
||||
for ($k = 100; $k < strlen($data); $k += 65536) {
|
||||
$data[$k] = chr(ord($data[$k]) ^ 0x55);
|
||||
}
|
||||
$out = fopen('php://temp', 'w+b');
|
||||
fwrite($out, $data);
|
||||
rewind($out);
|
||||
return $out;
|
||||
}
|
||||
|
||||
public function getRange(string $path, int $offset, int $length): string {
|
||||
$data = $this->inner->getRange($path, $offset, $length);
|
||||
if (str_starts_with($path, 'packs/') && $this->ranges > 0) {
|
||||
$this->ranges--;
|
||||
$data[0] = chr(ord($data[0]) ^ 0x55);
|
||||
}
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function exists(string $path): bool {
|
||||
return $this->inner->exists($path);
|
||||
}
|
||||
|
||||
public function move(string $from, string $to): void {
|
||||
$this->inner->move($from, $to);
|
||||
}
|
||||
|
||||
public function list(string $prefix): array {
|
||||
return $this->inner->list($prefix);
|
||||
}
|
||||
|
||||
public function listFiles(string $dir): array {
|
||||
return $this->inner->listFiles($dir);
|
||||
}
|
||||
|
||||
public function delete(string $path): void {
|
||||
$this->inner->delete($path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Job\BackupRun;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/** Folders without files are recorded in the snapshot's meta record and come back on a restore. */
|
||||
class EmptyDirsTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private LocalBackend $backend;
|
||||
private KeyRing $keys;
|
||||
private string $src;
|
||||
|
||||
protected function setUp(): void {
|
||||
$this->backend = new LocalBackend($this->tempDir());
|
||||
$this->keys = KeyRing::generate();
|
||||
Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$this->src = $this->tempDir();
|
||||
mkdir("$this->src/data/alice/files/Docs", 0700, true);
|
||||
mkdir("$this->src/data/alice/files/Hollow/Nested", 0700, true);
|
||||
mkdir("$this->src/data/skipped/inner", 0700, true);
|
||||
file_put_contents("$this->src/data/alice/files/Docs/note.txt", "hello\n");
|
||||
touch("$this->src/data/alice/files/Hollow/Nested", 1700000000);
|
||||
}
|
||||
|
||||
private function repo(): Repository {
|
||||
return Repository::openWithKey($this->backend, $this->keys);
|
||||
}
|
||||
|
||||
/** @param list<string> $exclude */
|
||||
private function backup(array $exclude = [], float $seconds = 30.0): string {
|
||||
$state = BackupRun::start(['data' => "$this->src/data"], null, '', $exclude);
|
||||
for ($i = 0; $i < 500 && $state['phase'] !== 'commit'; $i++) {
|
||||
$state = json_decode(json_encode(BackupRun::step($this->repo(), $state, microtime(true) + $seconds)), true);
|
||||
}
|
||||
$this->assertSame('commit', $state['phase']);
|
||||
$this->repo()->recordSnapshot($state['snapshot']);
|
||||
return $state['snapshot'];
|
||||
}
|
||||
|
||||
public function testOnlyTheDeepestEmptyFolderIsRecordedAndFilesStayAsTheyAre(): void {
|
||||
$snapshot = $this->backup();
|
||||
$paths = array_column($this->repo()->emptyDirs($snapshot), 0);
|
||||
$this->assertSame(['data/alice/files/Hollow/Nested', 'data/skipped/inner'], $paths);
|
||||
$this->assertSame(['data/alice/files/Docs/note.txt'], array_column(iterator_to_array($this->repo()->entries($snapshot), false), 'p'));
|
||||
$this->assertSame(2, $this->repo()->snapshotMeta($snapshot)['stats']['emptyDirs']);
|
||||
}
|
||||
|
||||
public function testAFolderWithOnlyExcludedContentCountsAsEmpty(): void {
|
||||
$snapshot = $this->backup(['data/skipped/inner']);
|
||||
$this->assertContains('data/skipped', array_column($this->repo()->emptyDirs($snapshot), 0));
|
||||
$this->assertNotContains('data/skipped/inner', array_column($this->repo()->emptyDirs($snapshot), 0));
|
||||
}
|
||||
|
||||
public function testAResumedWalkRecordsEveryFolderOnce(): void {
|
||||
for ($i = 0; $i < 30; $i++) {
|
||||
mkdir("$this->src/data/many/d$i", 0700, true);
|
||||
file_put_contents("$this->src/data/many/f$i", 'x');
|
||||
}
|
||||
$snapshot = $this->backup([], 0.0); // a step per entry
|
||||
$paths = array_column($this->repo()->emptyDirs($snapshot), 0);
|
||||
$this->assertCount(32, $paths);
|
||||
$this->assertSame(array_unique($paths), $paths);
|
||||
}
|
||||
|
||||
public function testANameASnapshotCannotHoldIsReportedAndTheSnapshotStaysReadable(): void {
|
||||
mkdir("$this->src/data/odd\\folder", 0700, true);
|
||||
mkdir("$this->src/data/alice/files/line\nbreak", 0700, true);
|
||||
file_put_contents("$this->src/data/alice/files/back\\slash.txt", 'x');
|
||||
file_put_contents("$this->src/data/alice/files/\xff.txt", 'x');
|
||||
$state = BackupRun::start(['data' => "$this->src/data"]);
|
||||
for ($i = 0; $i < 50 && $state['phase'] !== 'commit'; $i++) {
|
||||
// A step per entry, state through JSON: as separate processes would
|
||||
$state = json_decode(json_encode(BackupRun::step($this->repo(), $state, 0.0), JSON_THROW_ON_ERROR), true, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
$this->assertSame('commit', $state['phase']);
|
||||
$this->assertSame(4, $state['stats']['badNames'], 'each reported once, over several steps');
|
||||
$this->assertArrayNotHasKey('unreadable', $state['stats'], 'not a problem with permissions');
|
||||
$shown = BackupRun::warnings($state)['badNames'];
|
||||
$this->assertContains('data/alice/files/line?break', $shown, 'no control character in a message');
|
||||
$this->assertSame($shown, array_filter($shown, fn (string $name): bool => mb_check_encoding($name, 'UTF-8')));
|
||||
$this->repo()->recordSnapshot($state['snapshot']);
|
||||
|
||||
$this->assertSame(['data/alice/files/Hollow/Nested', 'data/skipped/inner'], array_column($this->repo()->emptyDirs($state['snapshot']), 0));
|
||||
$target = $this->tempDir();
|
||||
$this->assertSame(1, $this->repo()->restore($state['snapshot'], $target));
|
||||
$this->assertFileExists("$target/data/alice/files/Docs/note.txt");
|
||||
}
|
||||
|
||||
public function testAnEmptyFolderThatIsThePrefixItselfIsTheTarget(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$this->repo()->restore($snapshot, $target, 'data/skipped/inner/', true);
|
||||
$this->assertSame([], glob("$target/*"), 'not a folder "inner" inside the target');
|
||||
}
|
||||
|
||||
public function testRestoreBringsThemBackWithTheirTime(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$this->assertSame(1, $this->repo()->restore($snapshot, $target));
|
||||
$this->assertDirectoryExists("$target/data/alice/files/Hollow/Nested");
|
||||
clearstatcache();
|
||||
$this->assertSame(1700000000, filemtime("$target/data/alice/files/Hollow/Nested"));
|
||||
$this->assertDirectoryExists("$target/data/skipped/inner");
|
||||
}
|
||||
|
||||
public function testRestoreOfAPrefixStripsAndLeavesOthersOut(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$this->repo()->restore($snapshot, $target, 'data/alice/', true);
|
||||
$this->assertDirectoryExists("$target/files/Hollow/Nested");
|
||||
$this->assertDirectoryDoesNotExist("$target/skipped");
|
||||
$this->assertDirectoryDoesNotExist("$target/inner");
|
||||
}
|
||||
|
||||
public function testRunningAgainLeavesAFolderThatIsThereAlone(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$this->repo()->restore($snapshot, $target);
|
||||
touch("$target/data/alice/files/Hollow/Nested", 1800000000);
|
||||
file_put_contents("$target/data/alice/files/Hollow/Nested/added", 'kept');
|
||||
touch("$target/data/alice/files/Hollow/Nested", 1800000000);
|
||||
$this->repo()->restore($snapshot, $target, keepUnchanged: true);
|
||||
clearstatcache();
|
||||
$this->assertSame(1800000000, filemtime("$target/data/alice/files/Hollow/Nested"));
|
||||
$this->assertFileExists("$target/data/alice/files/Hollow/Nested/added");
|
||||
}
|
||||
|
||||
public function testALinkOutOfTheTargetIsRefused(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$outside = $this->tempDir();
|
||||
mkdir("$target/data/alice/files", 0700, true);
|
||||
symlink($outside, "$target/data/alice/files/Hollow");
|
||||
try {
|
||||
$this->repo()->restore($snapshot, $target);
|
||||
$this->fail('restored through a link');
|
||||
} catch (\OCA\NgBackup\Repository\RestoreTargetException $e) {
|
||||
$this->assertStringContainsString('outside', $e->getMessage());
|
||||
}
|
||||
$this->assertSame([], glob("$outside/*"));
|
||||
}
|
||||
|
||||
public function testASnapshotWithoutThemHasNone(): void {
|
||||
$src = $this->tempDir();
|
||||
file_put_contents("$src/a", 'x');
|
||||
$snapshot = $this->repo()->backupDirectory($src)['snapshot'];
|
||||
$this->assertSame([], $this->repo()->emptyDirs($snapshot));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Job\BackupRun;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/** Folders with something in them get their original modification time back on a restore. */
|
||||
class FolderTimesTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private const OLD = 1500000000;
|
||||
private const OLDER = 1400000000;
|
||||
|
||||
private LocalBackend $backend;
|
||||
private KeyRing $keys;
|
||||
private string $src;
|
||||
|
||||
protected function setUp(): void {
|
||||
$this->backend = new LocalBackend($this->tempDir());
|
||||
$this->keys = KeyRing::generate();
|
||||
Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$this->src = $this->tempDir();
|
||||
mkdir("$this->src/data/alice/files/Old/Deep", 0700, true);
|
||||
mkdir("$this->src/data/alice/files/2026", 0700, true);
|
||||
file_put_contents("$this->src/data/alice/files/Old/Deep/a.txt", 'a');
|
||||
file_put_contents("$this->src/data/alice/files/Old/b.txt", 'b');
|
||||
file_put_contents("$this->src/data/alice/files/2026/c.txt", 'c');
|
||||
touch("$this->src/data/alice/files/Old/Deep", self::OLDER);
|
||||
touch("$this->src/data/alice/files/Old", self::OLD);
|
||||
touch("$this->src/data/alice/files/2026", self::OLD);
|
||||
}
|
||||
|
||||
private function repo(): Repository {
|
||||
return Repository::openWithKey($this->backend, $this->keys);
|
||||
}
|
||||
|
||||
private function backup(float $seconds = 30.0): string {
|
||||
$state = BackupRun::start(['data' => "$this->src/data"]);
|
||||
for ($i = 0; $i < 500 && $state['phase'] !== 'commit'; $i++) {
|
||||
$state = json_decode(json_encode(BackupRun::step($this->repo(), $state, microtime(true) + $seconds)), true);
|
||||
}
|
||||
$this->assertSame('commit', $state['phase']);
|
||||
$this->repo()->recordSnapshot($state['snapshot']);
|
||||
return $state['snapshot'];
|
||||
}
|
||||
|
||||
private function time(string $path): int {
|
||||
clearstatcache();
|
||||
return (int)filemtime($path);
|
||||
}
|
||||
|
||||
public function testEachFolderIsRecordedOnceAlsoInSmallSteps(): void {
|
||||
$expected = ['data', 'data/alice', 'data/alice/files', 'data/alice/files/2026', 'data/alice/files/Old', 'data/alice/files/Old/Deep'];
|
||||
$this->assertSame($expected, array_column($this->repo()->folderTimes($this->backup()), 0));
|
||||
$this->assertSame($expected, array_column($this->repo()->folderTimes($this->backup(0.0)), 0), 'a step per entry');
|
||||
}
|
||||
|
||||
public function testNoEmptyFolderIsAmongThem(): void {
|
||||
mkdir("$this->src/data/alice/files/Hollow");
|
||||
$snapshot = $this->backup();
|
||||
$this->assertNotContains('data/alice/files/Hollow', array_column($this->repo()->folderTimes($snapshot), 0));
|
||||
$this->assertContains('data/alice/files/Hollow', array_column($this->repo()->emptyDirs($snapshot), 0));
|
||||
}
|
||||
|
||||
public function testARestoreBringsThemBack(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$this->repo()->restore($snapshot, $target);
|
||||
$this->assertSame(self::OLD, $this->time("$target/data/alice/files/Old"));
|
||||
$this->assertSame(self::OLDER, $this->time("$target/data/alice/files/Old/Deep"));
|
||||
$this->assertSame(self::OLD, $this->time("$target/data/alice/files/2026"));
|
||||
}
|
||||
|
||||
public function testARestoreThatGoesOnAfterAStopToo(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$this->repo()->restore($snapshot, $target);
|
||||
touch("$target/data/alice/files/Old", 1800000000);
|
||||
$this->repo()->restore($snapshot, $target, keepUnchanged: true);
|
||||
$this->assertSame(self::OLD, $this->time("$target/data/alice/files/Old"));
|
||||
}
|
||||
|
||||
public function testAPrefixLeavesTheTargetAndOthersAlone(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$this->repo()->restore($snapshot, $target, 'data/alice/files/Old/', true);
|
||||
$this->assertSame(self::OLDER, $this->time("$target/Deep"));
|
||||
$this->assertNotSame(self::OLD, $this->time($target), 'the directory that was chosen is not ours to change');
|
||||
$this->assertDirectoryDoesNotExist("$target/2026");
|
||||
}
|
||||
|
||||
public function testALinkOutOfTheTargetIsRefused(): void {
|
||||
$snapshot = $this->backup();
|
||||
$target = $this->tempDir();
|
||||
$outside = $this->tempDir();
|
||||
touch($outside, 1800000000);
|
||||
mkdir("$target/data/alice/files", 0700, true);
|
||||
symlink($outside, "$target/data/alice/files/Old");
|
||||
$this->expectException(\OCA\NgBackup\Repository\RestoreTargetException::class);
|
||||
$this->repo()->restore($snapshot, $target);
|
||||
}
|
||||
|
||||
public function testAFolderWithABadNameIsLeftOutAndWhatIsInItReported(): void {
|
||||
mkdir("$this->src/data/odd\\folder");
|
||||
file_put_contents("$this->src/data/odd\\folder/x", 'x');
|
||||
$state = BackupRun::start(['data' => "$this->src/data"]);
|
||||
for ($i = 0; $i < 50 && $state['phase'] !== 'commit'; $i++) {
|
||||
$state = json_decode(json_encode(BackupRun::step($this->repo(), $state, 0.0), JSON_THROW_ON_ERROR), true, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
$this->assertSame('commit', $state['phase']);
|
||||
$this->assertSame(1, $state['stats']['badNames']);
|
||||
$this->repo()->recordSnapshot($state['snapshot']);
|
||||
$this->assertNotContains("data/odd\\folder", array_column($this->repo()->folderTimes($state['snapshot']), 0));
|
||||
}
|
||||
|
||||
public function testASnapshotWithoutThemHasNone(): void {
|
||||
$snapshot = $this->repo()->backupDirectory($this->src)['snapshot'];
|
||||
$this->assertSame([], $this->repo()->folderTimes($snapshot));
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,24 @@ class IndexCacheTest extends TestCase {
|
||||
$this->assertCount(12, self::hashTree($target));
|
||||
}
|
||||
|
||||
/** Connecting a new server to a location with backups reports how far reading the index is. */
|
||||
public function testOpeningReportsIndexProgress(): void {
|
||||
$src = $this->tempDir();
|
||||
for ($i = 0; $i < 6; $i++) {
|
||||
file_put_contents("$src/f$i", random_bytes(3 * 1048576));
|
||||
}
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
$keys = KeyRing::generate();
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'))->backupDirectory($src);
|
||||
$files = count($backend->list('index'));
|
||||
$seen = [];
|
||||
Repository::openWithKey($backend, $keys, null, null, function (int $done, int $total) use (&$seen): void {
|
||||
$seen[] = [$done, $total];
|
||||
});
|
||||
$this->assertCount($files, $seen);
|
||||
$this->assertSame([$files, $files], end($seen));
|
||||
}
|
||||
|
||||
/** The database dump reads in a read-only transaction: a pack that fills up then must not write the cache yet. */
|
||||
public function testHeldCacheWritesWaitForRelease(): void {
|
||||
$cache = new class implements IIndexCache {
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Repository\IIndexCache;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/**
|
||||
* A listing of the location that comes back empty because a connection failed quietly (an SFTP
|
||||
* connection a Storage Box dropped) must not empty the index. In a 1 TB disaster recovery test a
|
||||
* pack read failed after 13 hours, the reload of the index got an empty listing, and the restore
|
||||
* stopped with "Blob not in index".
|
||||
*/
|
||||
class IndexListingTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private string $root;
|
||||
private KeyRing $keys;
|
||||
private string $src;
|
||||
private string $snapshot;
|
||||
|
||||
protected function setUp(): void {
|
||||
$this->root = $this->tempDir();
|
||||
$this->keys = KeyRing::generate();
|
||||
$this->src = $this->tempDir();
|
||||
for ($i = 0; $i < 12; $i++) {
|
||||
file_put_contents("$this->src/f$i", random_bytes(3 * 1048576));
|
||||
}
|
||||
$repo = Repository::initWithKey(new LocalBackend($this->root), $this->keys, $this->keys->wrap('pw'));
|
||||
$this->snapshot = $repo->backupDirectory($this->src)['snapshot'];
|
||||
}
|
||||
|
||||
private static function cache(): IIndexCache {
|
||||
return indexCache();
|
||||
}
|
||||
|
||||
public function testARestoreGoesOnAfterAPackReadFailedAndTheListingCameBackEmpty(): void {
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
$repo = Repository::openWithKey($backend, $this->keys);
|
||||
$backend->failPackReads = 1; // the read of a pack fails once
|
||||
$backend->emptyIndexListings = 1; // and listing the index right after comes back empty
|
||||
$target = $this->tempDir();
|
||||
|
||||
$this->assertSame(12, $repo->restore($this->snapshot, $target));
|
||||
|
||||
$this->assertSame(self::hashTree($this->src), self::hashTree($target));
|
||||
$this->assertSame(0, $backend->failPackReads, 'the failure happened');
|
||||
$this->assertSame(0, $backend->emptyIndexListings, 'the empty listing was seen');
|
||||
}
|
||||
|
||||
public function testAPackThatStaysUnreadableIsStillReportedAsSuch(): void {
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
$repo = Repository::openWithKey($backend, $this->keys);
|
||||
$backend->failPackReads = 1000;
|
||||
|
||||
try {
|
||||
$repo->restore($this->snapshot, $this->tempDir());
|
||||
$this->fail('the packs cannot be read');
|
||||
} catch (BackendException $e) {
|
||||
$this->assertStringStartsWith('Not found: packs/', $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public function testACheckWithAnEmptyListingIsAnErrorNotEveryBlobMissing(): void {
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
$repo = Repository::openWithKey($backend, $this->keys);
|
||||
$backend->dead = true;
|
||||
|
||||
$this->expectException(BackendException::class);
|
||||
$this->expectExceptionMessage('listed no index files');
|
||||
$repo->verify($this->snapshot);
|
||||
}
|
||||
|
||||
public function testRefreshingFetchesOnlyIndexFilesItHasNotLoaded(): void {
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
$repo = Repository::openWithKey($backend, $this->keys);
|
||||
$index = (new \ReflectionProperty($repo, 'index'))->getValue($repo);
|
||||
$index->downloaded = 0;
|
||||
|
||||
$index->refresh();
|
||||
|
||||
$this->assertSame(0, $index->downloaded, 'every index file is loaded already, also those whose blobs are elsewhere too');
|
||||
}
|
||||
|
||||
public function testARestoreGoesOnWhenTheLocationCannotBeListedAfterAFailedRead(): void {
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
$repo = Repository::openWithKey($backend, $this->keys);
|
||||
$backend->failPackReads = 1;
|
||||
$backend->dead = true; // listing and config look gone; reading packs works again
|
||||
$target = $this->tempDir();
|
||||
|
||||
$this->assertSame(12, $repo->restore($this->snapshot, $target));
|
||||
|
||||
$this->assertSame(self::hashTree($this->src), self::hashTree($target));
|
||||
}
|
||||
|
||||
public function testAnEmptyListingThatWasAHiccupIsListedAgain(): void {
|
||||
$cache = self::cache();
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
Repository::openWithKey($backend, $this->keys, $cache);
|
||||
$cached = $cache->data;
|
||||
$backend->emptyIndexListings = 1;
|
||||
|
||||
$again = Repository::openWithKey($backend, $this->keys, $cache);
|
||||
|
||||
$this->assertSame($cached, $cache->data);
|
||||
$this->assertSame(12, $again->restore($this->snapshot, $this->tempDir()));
|
||||
}
|
||||
|
||||
public function testAListingThatStaysEmptyWhileIndexFilesAreThereIsNotTaken(): void {
|
||||
$cache = self::cache();
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
Repository::openWithKey($backend, $this->keys, $cache);
|
||||
$cached = $cache->data;
|
||||
$backend->emptyIndexListings = 5; // the config answers, the listing keeps lying
|
||||
|
||||
try {
|
||||
Repository::openWithKey($backend, $this->keys, $cache);
|
||||
$this->fail('index files are there');
|
||||
} catch (BackendException $e) {
|
||||
$this->assertStringContainsString('is there', $e->getMessage());
|
||||
}
|
||||
$this->assertSame($cached, $cache->data);
|
||||
}
|
||||
|
||||
public function testARepositoryThatReallyBecameEmptyIsTakenAsEmpty(): void {
|
||||
$cache = self::cache();
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
Repository::openWithKey($backend, $this->keys, $cache);
|
||||
$this->assertNotSame([], array_filter($cache->data));
|
||||
// another server forgot the last snapshot and cleaned up: no packs and index files left
|
||||
foreach (array_merge($backend->list('index'), $backend->list('packs')) as $path) {
|
||||
$backend->delete($path);
|
||||
}
|
||||
|
||||
Repository::openWithKey($backend, $this->keys, $cache);
|
||||
|
||||
$this->assertSame([], array_filter($cache->data), 'the stale cache rows are gone');
|
||||
}
|
||||
|
||||
public function testOpeningWithAnEmptyListingKeepsTheCachedIndex(): void {
|
||||
$cache = self::cache();
|
||||
$backend = new FlakyBackend(new LocalBackend($this->root));
|
||||
Repository::openWithKey($backend, $this->keys, $cache);
|
||||
$cached = $cache->data;
|
||||
$this->assertNotSame([], $cached);
|
||||
|
||||
$backend->dead = true;
|
||||
try {
|
||||
Repository::openWithKey($backend, $this->keys, $cache);
|
||||
$this->fail('an empty listing of a repository with an index must not be taken as is');
|
||||
} catch (BackendException $e) {
|
||||
$this->assertStringContainsString('listed no index files', $e->getMessage());
|
||||
}
|
||||
$this->assertSame($cached, $cache->data, 'the cache is untouched');
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
function indexCache(): IIndexCache {
|
||||
return new class implements IIndexCache {
|
||||
public array $data = [];
|
||||
public function all(string $r): array { return $this->data[$r] ?? []; }
|
||||
public function put(string $r, string $p, array $e): void { $this->data[$r][$p] = $e; }
|
||||
public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } }
|
||||
public function putMany(string $r, array $packs): void { foreach ($packs as $p => $e) { $this->put($r, (string)$p, $e); } }
|
||||
public function replaceAll(string $r, array $packs): void { $this->data[$r] = []; $this->putMany($r, $packs); }
|
||||
};
|
||||
}
|
||||
|
||||
/** A location whose pack reads fail, and whose index listings come back empty, a set number of times. */
|
||||
final class FlakyBackend implements IBackend {
|
||||
use \OCA\NgBackup\Backend\GetsOneByOne;
|
||||
|
||||
public int $failPackReads = 0;
|
||||
public int $emptyIndexListings = 0;
|
||||
/** The connection is gone: also the repository config looks missing. */
|
||||
public bool $dead = false;
|
||||
|
||||
public function __construct(
|
||||
private IBackend $inner,
|
||||
) {
|
||||
}
|
||||
|
||||
public function put(string $path, $stream): void {
|
||||
$this->inner->put($path, $stream);
|
||||
}
|
||||
|
||||
public function get(string $path) {
|
||||
if (str_starts_with($path, 'packs/') && $this->failPackReads > 0) {
|
||||
$this->failPackReads--;
|
||||
throw new BackendException('Not found: ' . $path);
|
||||
}
|
||||
return $this->inner->get($path);
|
||||
}
|
||||
|
||||
public function getRange(string $path, int $offset, int $length): string {
|
||||
if (str_starts_with($path, 'packs/') && $this->failPackReads > 0) {
|
||||
$this->failPackReads--;
|
||||
throw new BackendException('Not found: ' . $path);
|
||||
}
|
||||
return $this->inner->getRange($path, $offset, $length);
|
||||
}
|
||||
|
||||
public function exists(string $path): bool {
|
||||
return !$this->dead && $this->inner->exists($path);
|
||||
}
|
||||
|
||||
public function move(string $from, string $to): void {
|
||||
$this->inner->move($from, $to);
|
||||
}
|
||||
|
||||
public function list(string $prefix): array {
|
||||
return $this->inner->list($prefix);
|
||||
}
|
||||
|
||||
public function listFiles(string $dir): array {
|
||||
if ($dir === 'index' && $this->dead) {
|
||||
return [];
|
||||
}
|
||||
if ($dir === 'index' && $this->emptyIndexListings > 0) {
|
||||
$this->emptyIndexListings--;
|
||||
return [];
|
||||
}
|
||||
return $this->inner->listFiles($dir);
|
||||
}
|
||||
|
||||
public function delete(string $path): void {
|
||||
$this->inner->delete($path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\CryptoException;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/** Disaster recovery without the recovery kit: the location gives its key with a passphrase. */
|
||||
class KeyFromLocationTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private string $dir;
|
||||
private LocalBackend $backend;
|
||||
private KeyRing $keys;
|
||||
|
||||
protected function setUp(): void {
|
||||
$this->dir = $this->tempDir();
|
||||
$this->backend = new LocalBackend($this->dir);
|
||||
$this->keys = KeyRing::generate();
|
||||
$slots = ['slots' => [
|
||||
['slot' => 1, 'label' => 'A', 'created' => 0, 'key' => $this->keys->wrap('first passphrase')],
|
||||
['slot' => 2, 'label' => 'B', 'created' => 0, 'key' => $this->keys->wrap('second passphrase')],
|
||||
]];
|
||||
Repository::initWithKey($this->backend, $this->keys, $slots)->recordSettings(['delete_delay_days' => 30]);
|
||||
}
|
||||
|
||||
public function testEachPassphraseGivesTheKeyThatOpensTheBackups(): void {
|
||||
foreach (['first passphrase', 'second passphrase'] as $passphrase) {
|
||||
$keys = KeyRing::unwrapAny(Repository::wrappedKeyAt($this->backend), $passphrase);
|
||||
$this->assertTrue(Repository::madeWithKey($this->backend, $keys));
|
||||
$this->assertSame($this->keys->exportMasterKey(), $keys->exportMasterKey());
|
||||
}
|
||||
}
|
||||
|
||||
public function testAWrongPassphraseGivesNothing(): void {
|
||||
$this->expectException(CryptoException::class);
|
||||
KeyRing::unwrapAny(Repository::wrappedKeyAt($this->backend), 'not the passphrase');
|
||||
}
|
||||
|
||||
public function testAnotherKeyIsNotTakenForTheRightOne(): void {
|
||||
$this->assertFalse(Repository::madeWithKey($this->backend, KeyRing::generate()));
|
||||
}
|
||||
|
||||
public function testTheSettingsAreSealedAndWrittenOnce(): void {
|
||||
$this->assertSame(30, Repository::settingsAt($this->backend, $this->keys)['delete_delay_days']);
|
||||
Repository::openWithKey($this->backend, $this->keys)->recordSettings(['delete_delay_days' => 1]);
|
||||
$this->assertSame(30, Repository::settingsAt($this->backend, $this->keys)['delete_delay_days']);
|
||||
$this->assertSame(['unreadable' => true], Repository::settingsAt($this->backend, KeyRing::generate()), 'another key: there, but not readable');
|
||||
unlink($this->dir . '/settings');
|
||||
$this->assertSame([], Repository::settingsAt($this->backend, $this->keys), 'none at all: a location from before 1.1.0');
|
||||
}
|
||||
|
||||
public function testAlteredSettingsAreNotBelieved(): void {
|
||||
$file = $this->dir . '/settings';
|
||||
$raw = file_get_contents($file);
|
||||
$raw[strlen($raw) - 1] = $raw[strlen($raw) - 1] ^ "\x01";
|
||||
file_put_contents($file, $raw);
|
||||
$this->assertSame(['unreadable' => true], Repository::settingsAt($this->backend, $this->keys));
|
||||
}
|
||||
|
||||
public function testTheKitWritesALostConfigAgainAndTheBackupsOpen(): void {
|
||||
$src = $this->tempDir();
|
||||
file_put_contents("$src/a", 'still here');
|
||||
$repo = Repository::openWithKey($this->backend, $this->keys);
|
||||
$id = $repo->id();
|
||||
$snapshot = $repo->backupDirectory($src)['snapshot'];
|
||||
unlink($this->dir . '/config');
|
||||
$this->assertTrue(Repository::holdsData($this->backend));
|
||||
|
||||
Repository::repairConfig($this->backend, $this->keys, $this->keys->wrap('kit passphrase'));
|
||||
$repo = Repository::openWithKey($this->backend, $this->keys);
|
||||
$this->assertSame($id, $repo->id());
|
||||
$this->assertSame([$snapshot], $repo->listSnapshots());
|
||||
$out = $this->tempDir();
|
||||
$repo->restore($snapshot, $out);
|
||||
$this->assertSame('still here', file_get_contents("$out/a"));
|
||||
$this->assertSame($this->keys->exportMasterKey(), KeyRing::unwrapAny(Repository::wrappedKeyAt($this->backend), 'kit passphrase')->exportMasterKey());
|
||||
}
|
||||
|
||||
public function testAnotherKeyWritesNoConfig(): void {
|
||||
unlink($this->dir . '/config');
|
||||
$other = KeyRing::generate();
|
||||
try {
|
||||
Repository::repairConfig($this->backend, $other, $other->wrap('x'));
|
||||
$this->fail('a config was written with another key');
|
||||
} catch (\OCA\NgBackup\Repository\RepositoryException $e) {
|
||||
$this->assertStringContainsString('another server', $e->getMessage());
|
||||
}
|
||||
$this->assertFileDoesNotExist($this->dir . '/config');
|
||||
}
|
||||
|
||||
public function testAConfigThatIsThereIsNeverRepaired(): void {
|
||||
$before = file_get_contents($this->dir . '/config');
|
||||
try {
|
||||
Repository::repairConfig($this->backend, $this->keys, $this->keys->wrap('x'));
|
||||
$this->fail('an existing config was replaced');
|
||||
} catch (\OCA\NgBackup\Repository\RepositoryException) {
|
||||
}
|
||||
$this->assertSame($before, file_get_contents($this->dir . '/config'));
|
||||
}
|
||||
|
||||
public function testALocationWithoutSettingsHasNone(): void {
|
||||
$other = new LocalBackend($this->tempDir());
|
||||
Repository::initWithKey($other, $this->keys, $this->keys->wrap('x'));
|
||||
$this->assertSame([], Repository::settingsAt($other, $this->keys));
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,33 @@ class PruneTest extends TestCase {
|
||||
$this->assertSame(self::hashTree($src), self::hashTree($target));
|
||||
}
|
||||
|
||||
public function testRestoreFollowsPacksRepackedByAnotherServer(): void {
|
||||
// A disaster recovery test on a second server reads the location while the original
|
||||
// server's weekly cleanup repacks it: the restore must not stop at a pack that moved.
|
||||
$src = $this->tempDir();
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
$keys = KeyRing::generate();
|
||||
$repo = Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
for ($i = 0; $i < 40; $i++) {
|
||||
file_put_contents("$src/f$i", random_bytes(500000));
|
||||
}
|
||||
$a = $repo->backupDirectory($src);
|
||||
for ($i = 0; $i < 40; $i++) {
|
||||
if ($i % 5 !== 0) {
|
||||
unlink("$src/f$i");
|
||||
}
|
||||
}
|
||||
$b = $repo->backupDirectory($src, $a['snapshot']);
|
||||
$reader = Repository::openWithKey($backend, $keys); // index loaded before the cleanup
|
||||
$repo->forget($a['snapshot']);
|
||||
$repo->purgeTrash(0);
|
||||
$this->assertGreaterThan(0, $repo->prune(false, 0.5)['repacked']);
|
||||
|
||||
$target = $this->tempDir();
|
||||
$reader->restore($b['snapshot'], $target);
|
||||
$this->assertSame(self::hashTree($src), self::hashTree($target));
|
||||
}
|
||||
|
||||
public function testUntrashRestoresAWorkingSnapshot(): void {
|
||||
$src = $this->tempDir();
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Backend\IReadsAhead;
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Repository\ReadAheadPlan;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
require_once __DIR__ . '/PackReadTest.php'; // PackOpenCountingBackend
|
||||
|
||||
/** A full restore reads the packs ahead (IReadsAhead) and falls back to the usual reads per range. */
|
||||
class ReadAheadTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private string $root;
|
||||
private KeyRing $keys;
|
||||
private string $src;
|
||||
private string $snapshot;
|
||||
|
||||
protected function setUp(): void {
|
||||
$this->root = $this->tempDir();
|
||||
$this->keys = KeyRing::generate();
|
||||
$this->src = $this->tempDir();
|
||||
// three packs of 32 MiB, and small files in between
|
||||
for ($i = 0; $i < 4; $i++) {
|
||||
file_put_contents(sprintf('%s/big%d', $this->src, $i), random_bytes(20 * 1048576));
|
||||
for ($j = 0; $j < 20; $j++) {
|
||||
file_put_contents(sprintf('%s/small%d-%02d', $this->src, $i, $j), random_bytes(30000));
|
||||
}
|
||||
}
|
||||
file_put_contents("$this->src/same", file_get_contents("$this->src/small0-00")); // the same blob again, later
|
||||
$repo = Repository::initWithKey(new LocalBackend($this->root), $this->keys, $this->keys->wrap('pw'));
|
||||
$this->snapshot = $repo->backupDirectory($this->src)['snapshot'];
|
||||
}
|
||||
|
||||
private function restore(ReadAheadBackend $backend, string $prefix = '', ?string $target = null): string {
|
||||
$target ??= $this->tempDir();
|
||||
$repo = Repository::openWithKey($backend, $this->keys, null, null);
|
||||
$repo->restore($this->snapshot, $target, $prefix);
|
||||
return $target;
|
||||
}
|
||||
|
||||
private function assertRestored(string $target, string $prefix = ''): void {
|
||||
$expected = array_filter(self::hashTree($this->src), fn (string $p): bool => str_starts_with($p, $prefix), ARRAY_FILTER_USE_KEY);
|
||||
$this->assertSame($expected, self::hashTree($target));
|
||||
$this->assertSame([], glob("$target/.ngb-restore-spool-*"), 'the spool is removed');
|
||||
}
|
||||
|
||||
public function testARestoreReadsEveryPackAhead(): void {
|
||||
$backend = new ReadAheadBackend(new LocalBackend($this->root));
|
||||
|
||||
$this->assertRestored($this->restore($backend));
|
||||
|
||||
$this->assertSame(0, $backend->counter->packOpens(), 'nothing read the usual way');
|
||||
$this->assertGreaterThanOrEqual(3, count($backend->asked));
|
||||
// in the order of the files, one range per pack the restore goes through (the same blob
|
||||
// again later is a range of its own)
|
||||
$this->assertLessThanOrEqual(count($backend->inner->list('packs')) + 1, count($backend->asked));
|
||||
}
|
||||
|
||||
public function testAPlanThatCannotBeMadeLeavesTheRestoreToTheUsualReads(): void {
|
||||
$backend = new ReadAheadBackend(new LocalBackend($this->root), failSnapshotReads: 1);
|
||||
|
||||
$this->assertRestored($this->restore($backend));
|
||||
|
||||
$this->assertSame([], $backend->asked, 'no reading ahead without a plan');
|
||||
$this->assertSame(2, $backend->snapshotReads, 'the restore itself read the snapshot again');
|
||||
}
|
||||
|
||||
public function testTheSpoolOfAStoppedRestoreIsRemoved(): void {
|
||||
$target = $this->tempDir();
|
||||
mkdir("$target/.ngb-restore-spool-0badc0de", 0700); // a restore that was killed
|
||||
mkdir("$target/.ngb-restore-spool-8badf00d", 0700); // ... while it wrote a range
|
||||
file_put_contents("$target/.ngb-restore-spool-8badf00d/smb-01234567", str_repeat('x', 4096));
|
||||
|
||||
$this->assertRestored($this->restore(new ReadAheadBackend(new LocalBackend($this->root)), '', $target));
|
||||
$this->assertSame([], glob("$target/.ngb-restore-spool-*"));
|
||||
}
|
||||
|
||||
public function testARangeThatCouldNotBeReadAheadIsReadTheUsualWay(): void {
|
||||
$backend = new ReadAheadBackend(new LocalBackend($this->root), fail: [1]);
|
||||
|
||||
$this->assertRestored($this->restore($backend));
|
||||
|
||||
$this->assertGreaterThan(0, $backend->counter->packOpens());
|
||||
}
|
||||
|
||||
public function testADamagedCopyIsReadAgainTheUsualWay(): void {
|
||||
$backend = new ReadAheadBackend(new LocalBackend($this->root), damage: [0, 2]);
|
||||
|
||||
$this->assertRestored($this->restore($backend));
|
||||
|
||||
$this->assertGreaterThan(0, $backend->counter->packOpens());
|
||||
}
|
||||
|
||||
public function testAPartialRestoreReadsOnlyWhatItNeeds(): void {
|
||||
$backend = new ReadAheadBackend(new LocalBackend($this->root));
|
||||
|
||||
$this->assertRestored($this->restore($backend, 'big1'), 'big1');
|
||||
|
||||
$this->assertSame(0, $backend->counter->packOpens());
|
||||
$this->assertCount(2, $backend->asked, 'the end of one pack and the start of the next');
|
||||
$total = array_sum(array_map(fn (array $r): int => $r[2], $backend->asked));
|
||||
$this->assertLessThan(21 * 1048576, $total, 'not the whole packs');
|
||||
}
|
||||
|
||||
public function testABackendThatCannotReadAheadNowRestoresAsBefore(): void {
|
||||
$backend = new ReadAheadBackend(new LocalBackend($this->root), unavailable: true);
|
||||
|
||||
$this->assertRestored($this->restore($backend));
|
||||
|
||||
$this->assertGreaterThan(0, $backend->counter->packOpens());
|
||||
}
|
||||
|
||||
public function testAFailingRestoreStillEndsTheReadsAheadAndRemovesTheSpool(): void {
|
||||
// the second range fails, and so does reading it the usual way
|
||||
$backend = new ReadAheadBackend(new LocalBackend($this->root), fail: [1], packsGone: true);
|
||||
$target = $this->tempDir();
|
||||
$repo = Repository::openWithKey($backend, $this->keys, null, null);
|
||||
|
||||
try {
|
||||
$repo->restore($this->snapshot, $target);
|
||||
$this->fail('restore should fail');
|
||||
} catch (BackendException $e) {
|
||||
$this->assertStringStartsWith('Not found', $e->getMessage());
|
||||
}
|
||||
|
||||
$this->assertTrue($backend->ended, 'the generator was ended');
|
||||
$this->assertSame([], glob("$target/.ngb-restore-spool-*"));
|
||||
}
|
||||
|
||||
public function testPlanJoinsBlobsThatFollowEachOtherInAPack(): void {
|
||||
$plan = new ReadAheadPlan('/nowhere');
|
||||
$plan->add('packs/aa/a', 0, 100);
|
||||
$plan->add('packs/aa/a', 100, 100);
|
||||
$plan->add('packs/aa/a', 50, 10); // the same blob again: inside the range
|
||||
$plan->add('packs/aa/a', 200 + ReadAheadPlan::GAP, 100); // a gap it reads along
|
||||
$plan->add('packs/aa/a', 300 + 3 * ReadAheadPlan::GAP, 100); // too far: a new range
|
||||
$plan->add('packs/bb/b', 0, 100);
|
||||
$plan->add('packs/bb/b', 0, 100); // backwards is fine as long as it is inside
|
||||
$plan->add('packs/aa/a', 0, 100); // back to an earlier pack: a new range
|
||||
|
||||
$this->assertSame([
|
||||
['packs/aa/a', 0, 300 + ReadAheadPlan::GAP],
|
||||
['packs/aa/a', 300 + 3 * ReadAheadPlan::GAP, 100],
|
||||
['packs/bb/b', 0, 100],
|
||||
['packs/aa/a', 0, 100],
|
||||
], $plan->ranges());
|
||||
}
|
||||
|
||||
public function testPlanSkipsRangesWhoseBlobsWereReadElsewhere(): void {
|
||||
$plan = new ReadAheadPlan('/nowhere');
|
||||
foreach (range(0, 4) as $p) {
|
||||
$plan->add("packs/0$p/p$p", 0, 10);
|
||||
}
|
||||
$plan->readFrom((function (): \Generator {
|
||||
foreach (range(0, 4) as $p) {
|
||||
$fh = fopen('php://temp', 'w+b');
|
||||
fwrite($fh, str_repeat((string)$p, 10));
|
||||
rewind($fh);
|
||||
yield $p => $fh;
|
||||
}
|
||||
})());
|
||||
|
||||
$this->assertSame('00000', $plan->read('packs/00/p0', 0, 5));
|
||||
$this->assertNull($plan->read('packs/77/moved', 0, 5), 'moved: not in the plan');
|
||||
$this->assertSame('333', $plan->read('packs/03/p3', 2, 3), 'the ranges in between were read elsewhere');
|
||||
$this->assertNull($plan->read('packs/01/p1', 0, 5), 'never backwards');
|
||||
$this->assertSame('44', $plan->read('packs/04/p4', 0, 2));
|
||||
}
|
||||
|
||||
public function testPlanKeepsARangeShorterThanTheMaximum(): void {
|
||||
$plan = new ReadAheadPlan('/nowhere');
|
||||
for ($o = 0; $o < 3 * ReadAheadPlan::MAX_RANGE; $o += 4 * 1048576) {
|
||||
$plan->add('packs/aa/a', $o, 4 * 1048576);
|
||||
}
|
||||
$this->assertCount(3, $plan->ranges());
|
||||
foreach ($plan->ranges() as $r) {
|
||||
$this->assertLessThanOrEqual(ReadAheadPlan::MAX_RANGE, $r[2]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads ahead from a local repository, one range after the other; can fail or damage ranges. */
|
||||
final class ReadAheadBackend implements IBackend, IReadsAhead {
|
||||
public PackOpenCountingBackend $counter;
|
||||
/** @var list<array{0:string, 1:int, 2:int}> */
|
||||
public array $asked = [];
|
||||
public bool $ended = false;
|
||||
|
||||
/**
|
||||
* @param list<int> $fail ranges given as null
|
||||
* @param list<int> $damage ranges given with a flipped byte in every blob
|
||||
*/
|
||||
public function __construct(
|
||||
public LocalBackend $inner,
|
||||
private array $fail = [],
|
||||
private array $damage = [],
|
||||
private bool $unavailable = false,
|
||||
/** reading a pack the usual way fails */
|
||||
private bool $packsGone = false,
|
||||
/** this many reads of a snapshot fail, then they work */
|
||||
private int $failSnapshotReads = 0,
|
||||
) {
|
||||
$this->counter = new PackOpenCountingBackend($inner);
|
||||
}
|
||||
|
||||
public function canReadAhead(): bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
public function readAhead(array $ranges, string $spool): ?\Generator {
|
||||
if ($this->unavailable) {
|
||||
return null;
|
||||
}
|
||||
\PHPUnit\Framework\Assert::assertDirectoryExists($spool);
|
||||
$this->asked = $ranges;
|
||||
return (function () use ($ranges): \Generator {
|
||||
try {
|
||||
foreach ($ranges as $i => [$path, $offset, $length]) {
|
||||
if (in_array($i, $this->fail, true)) {
|
||||
yield $i => null;
|
||||
continue;
|
||||
}
|
||||
$data = $this->inner->getRange($path, $offset, $length);
|
||||
if (in_array($i, $this->damage, true)) {
|
||||
for ($k = 0; $k < strlen($data); $k += 4096) {
|
||||
$data[$k] = chr(ord($data[$k]) ^ 1);
|
||||
}
|
||||
}
|
||||
$fh = fopen('php://temp', 'w+b');
|
||||
fwrite($fh, $data);
|
||||
rewind($fh);
|
||||
yield $i => $fh;
|
||||
fclose($fh);
|
||||
}
|
||||
} finally {
|
||||
$this->ended = true;
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
public function put(string $path, $stream): void {
|
||||
$this->counter->put($path, $stream);
|
||||
}
|
||||
|
||||
public int $snapshotReads = 0;
|
||||
|
||||
public function get(string $path) {
|
||||
if (str_starts_with($path, 'snapshots/')) {
|
||||
$this->snapshotReads++;
|
||||
if ($this->snapshotReads <= $this->failSnapshotReads) {
|
||||
throw new BackendException('Connection lost reading ' . $path);
|
||||
}
|
||||
}
|
||||
if ($this->packsGone && str_starts_with($path, 'packs/')) {
|
||||
throw new BackendException('Not found: ' . $path);
|
||||
}
|
||||
return $this->counter->get($path);
|
||||
}
|
||||
|
||||
public function getMany(array $paths): \Generator {
|
||||
return $this->counter->getMany($paths);
|
||||
}
|
||||
|
||||
public function getRange(string $path, int $offset, int $length): string {
|
||||
return $this->counter->getRange($path, $offset, $length);
|
||||
}
|
||||
|
||||
public function getRanges(array $ranges): \Generator {
|
||||
return $this->counter->getRanges($ranges);
|
||||
}
|
||||
|
||||
public function exists(string $path): bool {
|
||||
return $this->counter->exists($path);
|
||||
}
|
||||
|
||||
public function move(string $from, string $to): void {
|
||||
$this->counter->move($from, $to);
|
||||
}
|
||||
|
||||
public function list(string $prefix): array {
|
||||
return $this->counter->list($prefix);
|
||||
}
|
||||
|
||||
public function listFiles(string $dir): array {
|
||||
return $this->counter->listFiles($dir);
|
||||
}
|
||||
|
||||
public function delete(string $path): void {
|
||||
$this->counter->delete($path);
|
||||
}
|
||||
}
|
||||
@@ -87,6 +87,74 @@ class RepositoryTest extends TestCase {
|
||||
$this->assertSame(10, $repo->restore($r['snapshot'], $target, 'docs/d1/'));
|
||||
}
|
||||
|
||||
public function testRestoreOfOneFile(): void {
|
||||
$repo = Repository::init($this->backend, 'pw');
|
||||
$r = $repo->backupDirectory($this->src);
|
||||
$target = $this->tempDir();
|
||||
// occ backup:restore --to-directory passes the path with a "/" appended
|
||||
$this->assertSame(1, $repo->restore($r['snapshot'], $target, 'docs/d1/note-1.txt/'));
|
||||
$this->assertSame(file_get_contents($this->src . '/docs/d1/note-1.txt'), file_get_contents($target . '/docs/d1/note-1.txt'));
|
||||
$this->assertSame(0, $repo->restore($r['snapshot'], $this->tempDir(), 'docs/d1/note-1/'), 'a prefix of a name is not that file');
|
||||
|
||||
$stripped = $this->tempDir();
|
||||
$this->assertSame(1, $repo->restore($r['snapshot'], $stripped, 'docs/d1/note-1.txt/', true));
|
||||
$this->assertFileExists($stripped . '/note-1.txt');
|
||||
}
|
||||
|
||||
public function testARestoreThatRunsAgainKeepsWhatIsInPlaceAndRestoresTheRest(): void {
|
||||
$location = $this->tempDir();
|
||||
$backend = new LocalBackend($location);
|
||||
$repo = Repository::init($backend, 'pw');
|
||||
$r = $repo->backupDirectory($this->src);
|
||||
$target = $this->tempDir();
|
||||
$repo->restore($r['snapshot'], $target);
|
||||
|
||||
// as a stopped restore leaves it: one file not there yet, one with other content
|
||||
unlink($target . '/docs/d1/note-1.txt');
|
||||
file_put_contents($target . '/docs/d0/note-0.txt', 'something else');
|
||||
$this->assertSame(34, $repo->restore($r['snapshot'], $target, keepUnchanged: true));
|
||||
$this->assertSame(self::hashTree($this->src), self::hashTree($target));
|
||||
|
||||
// all in place: nothing is read from the location, which could not give it anyway now
|
||||
self::removeTree($location . '/packs');
|
||||
$this->assertSame(34, Repository::open($backend, 'pw')->restore($r['snapshot'], $target, keepUnchanged: true));
|
||||
$this->expectException(\Throwable::class);
|
||||
Repository::open($backend, 'pw')->restore($r['snapshot'], $target);
|
||||
}
|
||||
|
||||
private static function removeTree(string $dir): void {
|
||||
foreach (new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir, \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST) as $f) {
|
||||
$f->isDir() ? rmdir($f->getPathname()) : unlink($f->getPathname());
|
||||
}
|
||||
rmdir($dir);
|
||||
}
|
||||
|
||||
public function testRestoreRemovesTemporariesOfAStoppedRestore(): void {
|
||||
$repo = Repository::init($this->backend, 'pw');
|
||||
$r = $repo->backupDirectory($this->src);
|
||||
$target = $this->tempDir();
|
||||
mkdir($target . '/docs/d1', 0700, true);
|
||||
mkdir($target . '/elsewhere', 0700, true);
|
||||
$stale = [
|
||||
$target . '/docs/d1/note-1.txt.ngb-restore-0123456789ab', // from a restore to a directory
|
||||
$target . '/docs/d1/.note-4.txt.ngb-restore-a1b2c3', // from a restore through Nextcloud
|
||||
];
|
||||
$kept = [
|
||||
$target . '/docs/d1/notes.ngb-restore-draft.txt', // not a temporary of ours
|
||||
$target . '/elsewhere/x.ngb-restore-0123456789ab', // not a directory this restore writes to
|
||||
];
|
||||
foreach ([...$stale, ...$kept] as $f) {
|
||||
file_put_contents($f, 'partial');
|
||||
}
|
||||
$repo->restore($r['snapshot'], $target, 'docs/');
|
||||
foreach ($stale as $f) {
|
||||
$this->assertFileDoesNotExist($f);
|
||||
}
|
||||
foreach ($kept as $f) {
|
||||
$this->assertFileExists($f);
|
||||
}
|
||||
}
|
||||
|
||||
public function testWrongPassphrase(): void {
|
||||
Repository::init($this->backend, 'pw');
|
||||
$this->expectException(CryptoException::class);
|
||||
@@ -99,6 +167,20 @@ class RepositoryTest extends TestCase {
|
||||
Repository::init($this->backend, 'pw');
|
||||
}
|
||||
|
||||
public function testInitRefusesALocationWithBackupDataButNoConfig(): void {
|
||||
$repo = Repository::init($this->backend, 'pw');
|
||||
$repo->backupDirectory($this->src);
|
||||
$this->backend->delete('config'); // missing, or not seen for a moment
|
||||
|
||||
try {
|
||||
Repository::init($this->backend, 'pw');
|
||||
$this->fail('a new repository over existing backup data');
|
||||
} catch (RepositoryException $e) {
|
||||
$this->assertStringContainsString('holds backup data', $e->getMessage());
|
||||
}
|
||||
$this->assertFalse($this->backend->exists('config'), 'no new config written');
|
||||
}
|
||||
|
||||
public function testTamperedPackIsDetected(): void {
|
||||
$repo = Repository::init($this->backend, 'pw');
|
||||
$r = $repo->backupDirectory($this->src);
|
||||
|
||||