Compare commits
45 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4fa7cbf6ef | |||
| f4abfe0a8d | |||
| fe1a289c2d | |||
| ab3265e42c | |||
| 466cff8857 | |||
| 6bba0b59fc | |||
| 62766e1e71 | |||
| bfd54447b0 | |||
| 0b50bb6b81 | |||
| f415bab519 | |||
| 27eb0b32fc | |||
| b1d4f6a469 | |||
| fe7ad9b4b5 | |||
| 695d972785 | |||
| 972a52e665 | |||
| 7c08a10621 | |||
| 209520165e | |||
| 9554e774e5 | |||
| 8b2dca28e3 | |||
| 5742f77253 | |||
| 550249570b | |||
| 0eb23f56e7 | |||
| a0c1f1cd58 | |||
| 26cc9a897e | |||
| fe0385ad8c | |||
| b37a953cfd | |||
| b89b5b5fa6 | |||
| 16e09f4fde | |||
| 45069994e9 | |||
| 0f4a238414 | |||
| f39ca71dab | |||
| e8ecb77df5 | |||
| 70965dd4bd | |||
| cc09d74942 | |||
| eb9100a74d | |||
| 614caf9f63 | |||
| 943c95bf07 | |||
| ba7f70e0f8 | |||
| d70a869070 | |||
| c9f82a79e4 | |||
| 932f60bc5e | |||
| 935c7b6a08 | |||
| f18b4f0862 | |||
| 6cbc5d79d4 | |||
| 58711b67ef |
@@ -17,6 +17,8 @@ jobs:
|
||||
php: ['8.2', '8.3', '8.4']
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: ${{ matrix.php }}
|
||||
@@ -29,6 +31,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: '8.2'
|
||||
@@ -48,6 +52,8 @@ jobs:
|
||||
database: ['sqlite', 'mariadb', 'pgsql']
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Round trip on Nextcloud ${{ matrix.nextcloud }} / ${{ matrix.database }}
|
||||
run: tests/integration/matrix.sh "${{ matrix.nextcloud }}" "${{ matrix.database }}"
|
||||
|
||||
@@ -58,5 +64,43 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Round trip against S3 and SFTP
|
||||
run: tests/integration/targets-matrix.sh
|
||||
|
||||
integration-user:
|
||||
# Per-user backup/restore via user_migration (occ backup:user:backup / backup:user:restore),
|
||||
# an optional dependency installed from the app store. It calls user_migration's internal
|
||||
# UserMigrationService, whose API can change between Nextcloud versions, so this runs on every
|
||||
# supported version (not the database, which the main integration job already covers).
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
nextcloud: ['32', '33', '34', '35']
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up a fresh Nextcloud instance
|
||||
run: |
|
||||
docker run -d --name ngbu -e SQLITE_DATABASE=nextcloud -e NEXTCLOUD_ADMIN_USER=admin \
|
||||
-e NEXTCLOUD_ADMIN_PASSWORD=admin-user-roundtrip -e NEXTCLOUD_TRUSTED_DOMAINS=localhost \
|
||||
-v "$PWD:/var/www/html/custom_apps/ng_backup:ro" nextcloud:${{ matrix.nextcloud }}-apache
|
||||
for i in $(seq 1 90); do docker exec -u www-data ngbu php occ status 2>/dev/null | grep -q "installed: true" && break; sleep 4; done
|
||||
docker exec ngbu chown www-data:www-data /var/www/html/custom_apps
|
||||
- name: Per-user backup/restore round trip on Nextcloud ${{ matrix.nextcloud }}
|
||||
run: tests/integration/user-roundtrip.sh "docker exec -u www-data ngbu" /var/www/html/data
|
||||
|
||||
integration-full-restore:
|
||||
# Disaster recovery (occ backup:restore:full) on a SECOND, fresh Nextcloud instance that
|
||||
# never saw the first one, sharing only the backup location. Does not depend on the database
|
||||
# beyond what the main integration job already covers, so this runs once, on one NC version.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Full restore round trip
|
||||
run: tests/integration/full-restore-roundtrip.sh 35-apache
|
||||
|
||||
+101
-3
@@ -2,9 +2,101 @@
|
||||
|
||||
All notable changes to NG Backup are documented here. The format follows
|
||||
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows the stages in
|
||||
`PLAN.md` section 9. The `<version>` in `appinfo/info.xml` is plain SemVer-numeric (Nextcloud's
|
||||
own `info.xsd` doesn't allow a `-alpha`/`-beta`/`-rc` suffix there); the stability stage is instead
|
||||
carried by the git tag, the release, and this file.
|
||||
`PLAN.md` section 9. The `<version>` in `appinfo/info.xml` is plain SemVer-numeric on purpose:
|
||||
the app store accepts a `-beta` suffix, but Nextcloud servers then only offer the release on the
|
||||
beta/daily update channel, which would hide the app from most admins. The stability stage is
|
||||
instead carried by the app description, the git tag, the release, and this file.
|
||||
|
||||
## [0.9.3-beta.4] - 2026-10-08
|
||||
|
||||
Bugfix release.
|
||||
|
||||
### Fixed
|
||||
- A backup failed with "cannot execute INSERT in a read-only transaction" on PostgreSQL as soon
|
||||
as the database dump filled a whole pack (32 MiB): the pack's index cache row was written
|
||||
inside the read-only dump transaction. On other databases that row was rolled back silently.
|
||||
- Backups to a remote location slowed down as the repository grew: every backup step listed the
|
||||
repository's index with a round trip per file. Over SFTP to a Hetzner Storage Box a 60 GiB
|
||||
repository already lost a large part of each step to it; backing up continuously
|
||||
(`occ backup:run`) went from 45 KiB/s to 5.8 MB/s with the fix.
|
||||
|
||||
## [0.9.2-beta.3] - 2026-10-08
|
||||
|
||||
Bugfix release.
|
||||
|
||||
### Added
|
||||
- README: notes on using a Hetzner Storage Box over WebDAV and SMB.
|
||||
|
||||
### Fixed
|
||||
- Adding a backup location in the web interface failed with "Request failed with status code 403"
|
||||
when the login was more than 30 minutes old: the form did not ask for password confirmation.
|
||||
- Settings page: the "Backup and restore" / "Settings" tabs line up with the sections below them.
|
||||
- Backups: the runs table scrolls horizontally on narrow screens instead of running off the page.
|
||||
- `occ backup:verify --deep`: a blob that could not be read (network or storage error) was reported
|
||||
as "corrupted or tampered". Reads are now retried, and a read that keeps failing is reported
|
||||
separately as unreadable, with the error (run the check again; if the same blobs stay unreadable,
|
||||
their pack may be damaged or truncated). Only a blob that was read but fails decryption or its
|
||||
hash check counts as corrupted. Found on a Hetzner Storage Box over WebDAV, where the blob
|
||||
turned out to be intact.
|
||||
|
||||
## [0.9.1-beta.2] - 2026-10-07
|
||||
|
||||
Second beta, the first release in the Nextcloud app store (signed).
|
||||
|
||||
### Added
|
||||
- App store listing: screenshots, website, repository and documentation link in `appinfo/info.xml`.
|
||||
|
||||
### Fixed
|
||||
- Restore: the "new folder" option showed `"Restored "` without the date pattern.
|
||||
- Restore: the restore-point dropdown cut off the time.
|
||||
- Settings: the passphrase actions are on one line instead of stacked.
|
||||
- `docs/HOWTO.md`: the SFTP example used a non-existent auth mechanism (`sftp::password`
|
||||
instead of `password::password`).
|
||||
|
||||
## [0.9.0-beta.1] - 2026-10-07
|
||||
|
||||
First beta. Tested thoroughly and usable **alongside** your current backup, but **not yet a
|
||||
drop-in replacement** for it. We test what we can ourselves (Nextcloud 32–35, SQLite, MariaDB and
|
||||
PostgreSQL, S3, SFTP, FTP, SMB and NFS), but we need field experience from other setups: please
|
||||
report what works and what doesn't, especially on large instances and on storage providers not
|
||||
listed in the README.
|
||||
|
||||
### Added
|
||||
- Disaster recovery: `occ backup:key:import` and `occ backup:restore:full` restore the database,
|
||||
data directory and config onto a fresh installation of the same major version.
|
||||
- Per-user backup and restore via user_migration: `occ backup:user:backup` and
|
||||
`occ backup:user:restore` (replace the account after a safety export, or restore as
|
||||
`<uid>-bak`), also onto a server where the user does not exist yet. The commands ask before
|
||||
installing user_migration (or take `--install-user-migration`).
|
||||
- Checksum audit: `occ backup:verify`, shallow or `--deep`, for snapshots and per-user exports.
|
||||
- Retention for per-user exports: `occ backup:retention --user-last=N`.
|
||||
- FTP and SMB covered by the integration tests; NFS through the `local` backend on a mounted share.
|
||||
- `docs/HOWTO.md`: step by step from setup to a tested restore.
|
||||
- Warnings when a backup finishes with unreadable files or folders, or with files that kept
|
||||
changing while being read (log, `occ`, notification, snapshot).
|
||||
|
||||
### Changed
|
||||
- `backup:restore:full` refuses on an installation with more than one user and asks first;
|
||||
`backup:user:restore --mode=replace` asks before deleting an existing account (`--force` for
|
||||
scripts in both cases).
|
||||
- A failed scheduled backup is retried at most three times a day, an hour apart.
|
||||
- The recovery kit carries the deletion delay; the kit file is created owner-only.
|
||||
- Adding a location through the web API needs password confirmation.
|
||||
|
||||
### Fixed
|
||||
- Exports of users whose id contains `.part-` could be deleted by cleanup on remote locations.
|
||||
- Cleanup no longer frees the data of a per-user export whose manifest is missing.
|
||||
- Crash safety of moving snapshots to and from the trash, of catalog writes by concurrent
|
||||
backups and per-user exports, and of short uploads.
|
||||
- A file that keeps changing no longer makes a backup restart forever.
|
||||
- Long restores and exports keep their lock on the location alive.
|
||||
- Floating-point database columns are dumped exactly.
|
||||
|
||||
### Known limitations
|
||||
See "Known limitations" in the README: memory use on very large instances, never two servers
|
||||
writing to one location, removing a passphrase slot does not revoke access, no rollback detection
|
||||
before a disaster recovery, and a few smaller points. None of them block the road to 1.0.0; they
|
||||
are on the backlog in `PLAN.md`.
|
||||
|
||||
## [0.1.0-alpha.1] - 2026-10-05
|
||||
|
||||
@@ -42,3 +134,9 @@ and not implemented yet.
|
||||
- No FTP/SMB coverage in CI yet, and no verification/checksum-audit command yet (phase 2).
|
||||
- See the "Storage backend experiences" section of the README for provider-specific quirks found
|
||||
so far (e.g. Hetzner Storage Box sub-accounts).
|
||||
|
||||
[0.9.3-beta.4]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.3-beta.4
|
||||
[0.9.2-beta.3]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.2-beta.3
|
||||
[0.9.1-beta.2]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.1-beta.2
|
||||
[0.9.0-beta.1]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.0-beta.1
|
||||
[0.1.0-alpha.1]: https://github.com/dillardblom/ng_backup/releases/tag/v0.1.0-alpha.1
|
||||
|
||||
@@ -276,7 +276,7 @@ onderdelen wel en niet terugkomen, op basis van de geregistreerde migrators.
|
||||
| `backup:list [--target=]` | herstelpunten |
|
||||
| `backup:verify <snapshot> [--deep]` | checksums controleren, optioneel alles ontsleutelen |
|
||||
| `backup:restore:file <snapshot> <user> <pad> [--to=]` | bestand/map terugzetten |
|
||||
| `backup:restore:user <snapshot> <user> [--to-user=]` | gebruiker terugzetten via user_migration (fase 2) |
|
||||
| `backup:user:restore <snapshot> <user> [--to-user=]` | gebruiker terugzetten via user_migration (fase 2) |
|
||||
| `backup:restore:full <snapshot>` | volledige restore (fase 2, verse installatie) |
|
||||
| `backup:target:add/list/remove/test` | opslagdoelen beheren en testen |
|
||||
| `backup:key:export / import` | herstelkit |
|
||||
@@ -338,6 +338,49 @@ een backup-app die niet betrouwbaar is, is erger dan geen backup-app.
|
||||
Volgorde: fase 1 grondig testen, reviewen en debuggen; dan nadenken over 0.1.0-alpha.1. Een
|
||||
minimale volledige restore (`occ backup:restore:full` op een verse installatie) komt vóór de beta.
|
||||
|
||||
**Roadmap-indeling fase 2 (besluit 2026-10-05):** naar v1.2.0 toewerken met in fase 2:
|
||||
1. `backup:restore:full` (disaster recovery, verse installatie) — **klaar** (2026-10-05)
|
||||
2. FTP/SMB-ondersteuning — **klaar** (2026-10-06): beide liepen al via files_external's generieke
|
||||
adapter zonder enige ng_backup-code; alleen testdekking toegevoegd
|
||||
(`tests/integration/targets-matrix.sh`) + bevindingen in README ("Storage backend experiences").
|
||||
SMB vereist de `smbclient` PECL-extensie (niet standaard in de Nextcloud-image, in CI nu
|
||||
on-the-fly gebouwd). NFS is geen eigen backend: lokaal mounten + `local`-backend erop richten.
|
||||
3. Verificatie/checksum-audit-commando — **klaar** (2026-10-05)
|
||||
4. Retentie/prune voor user-exports — **klaar** (2026-10-06): `occ backup:retention --user-last=N`
|
||||
(standaard 3, per installatie, net als de bestaande snapshot-policy), toegepast door
|
||||
`occ backup:prune` (geen prullenbak voor user-exports, in tegenstelling tot snapshots — on-demand,
|
||||
niet het primaire vangnet).
|
||||
5. Automatische user-export in het reguliere schema — **naar backlog** (niet voor deze release; on-demand blijft)
|
||||
9. SMB-roundtrip (protocol, niet een specifieke server) — **klaar** (2026-10-06): CI-matrix, 2 GB-roundtrip en
|
||||
user-export via SMB; NFS via de OMV-share getest (`local`-backend op een gemounte share, 2026-10-06)
|
||||
10. `backup:user:restore` zonder user_migration: vraagt om toestemming om de app te installeren en te activeren (of
|
||||
`--install-user-migration`) — **klaar** (2026-10-06); waarschuwing over de settings-allowlist — **klaar**;
|
||||
agenda/contacten bij migratie naar een server zonder die gebruiker — **getest** (2026-10-06, naar `migrated-personal`/`migrated-contacts`)
|
||||
Backlog: (a) de originele weergavenaam herstellen (het label uit de export is niet getest als
|
||||
weergavenaam); (b) de interne `migrated-`-naam hernoemen (vereist directe databasewijziging, niet voor de beta).
|
||||
|
||||
**Backlog uit de pre-beta review (2026-10-07)** — geen belemmering voor 0.9.0-beta of 1.0.0, staat
|
||||
in README "Known limitations":
|
||||
1. Geheugen bij grote instanties (>1 TB, miljoenen bestanden): blob-index, prune-set, browse van een
|
||||
snapshot in de web-UI, `trashExtras` bij restore en het manifest van een user-export staan volledig
|
||||
in het geheugen; streaming/paginering nodig.
|
||||
2. Twee servers op één locatie: catalogus kan breken (generatie overschreven, of anchor >200 generaties
|
||||
achter). Docs-waarschuwing staat er; mogelijk later een echte put-if-absent per backend.
|
||||
3. Sleutelrotatie: een slot verwijderen trekt geen toegang in (master-key blijft gelijk).
|
||||
4. Rollback vóór DR niet detecteerbaar: referentie-generatie en -hash in de kit opnemen of tonen.
|
||||
5. Kit-velden buiten `wrapped_key` (o.a. `delete_delay_days`) zijn niet geauthenticeerd; nu alleen getoond.
|
||||
6. Repository-id in de AD van objecten (formaatwijziging).
|
||||
7. Wachtwoorden van locaties via `--option-file` of omgevingsvariabele i.p.v. de commandoregel.
|
||||
8. Tijdelijke restore-bestanden niet via de Files-API verwijderen (komen nu in de prullenbak).
|
||||
9. DB-restore in foreign-key-volgorde (cascades).
|
||||
10. Packs zonder index (index-upload mislukt) opruimen.
|
||||
11. `backup:restore:full` in onderhoudsmodus en hervatbaar maken; nu: opnieuw draaien met `--force`.
|
||||
12. Config/rename op FTP/SMB-servers die geen rename over een bestaand bestand toestaan.
|
||||
|
||||
Doorgeschoven naar **v1.3.0** (fase 3, niet meer fase 2): in-place volledige restore
|
||||
(eigen restore-modus i.p.v. onderhoudsmodus) en zelfbediening voor gebruikers (eigen bestanden
|
||||
terugzetten vanuit de Files-app).
|
||||
|
||||
**Let op (ontdekt 2026-10-05):** Nextcloud's eigen `info.xsd` staat voor `<version>` alleen
|
||||
`[0-9]+(\.[0-9]+){0,2}` toe — geen `-alpha`/`-beta`/`-rc`-achtervoegsel. De stabiliteitsaanduiding
|
||||
uit deze tabel gaat dus niet in `info.xml` (dat blijft gewoon oplopend, bijv. `0.1.0`), maar in de
|
||||
|
||||
@@ -2,10 +2,14 @@
|
||||
|
||||
Encrypted, incremental backups of Nextcloud (database, configuration and data) with restore
|
||||
from the web interface. Works without shell access: no external programs, no extra PHP
|
||||
extensions beyond sodium. Runs on Nextcloud All-in-One, regular installations and hosted
|
||||
Nextcloud.
|
||||
extensions beyond sodium for ng_backup itself. Runs on Nextcloud All-in-One, regular
|
||||
installations and hosted Nextcloud. (Choosing SMB as a storage backend is the one exception:
|
||||
see "Storage backend experiences" below — that requirement comes from files_external, not from
|
||||
ng_backup, and applies to any Nextcloud app that writes to an SMB share.)
|
||||
|
||||
**Status: in development (phase 1). Not ready for production use.** See `PLAN.md` for the design.
|
||||
**Status: beta (0.9.3).** Tested thoroughly, usable alongside your current backup, but not yet a
|
||||
drop-in replacement for it. See "Known limitations" below, `PLAN.md` for the design and
|
||||
`docs/HOWTO.md` for a step-by-step guide from setup to a tested restore.
|
||||
|
||||
License: AGPL-3.0-or-later.
|
||||
|
||||
@@ -28,7 +32,7 @@ License: AGPL-3.0-or-later.
|
||||
occ backup:key:init --label="Safe" # first passphrase, deletion delay (default 7 days)
|
||||
occ backup:key:slot:add --label="CTO" # optional second and third passphrase
|
||||
occ backup:key:slot:add --label="Head of IT"
|
||||
occ backup:key:kit --output=/root/ng_backup-kit.json
|
||||
occ backup:key:kit --output=/var/tmp/ng_backup-kit.json
|
||||
occ backup:key:confirm
|
||||
occ backup:target:add offsite amazons3 -a amazons3::accesskey -o bucket=... -o key=... -o secret=...
|
||||
occ backup:run
|
||||
@@ -59,7 +63,8 @@ All backups of an installation are encrypted with one master key.
|
||||
a single slot is possible, but then that one passphrase is a single point of failure.
|
||||
- When a holder leaves, give their slot a new passphrase (`occ backup:key:slot:replace`), download
|
||||
the new kit and confirm it. Note: an old kit together with the old passphrase still opens the
|
||||
same master key; to revoke it completely, start new locations with a new key.
|
||||
same master key; to revoke it completely, start new locations with a new key. This is noted on
|
||||
the backlog and will be addressed in a future version (key rotation).
|
||||
- Downloading the kit, and every change to the slots, requires the administrator's password, is
|
||||
written to the audit log and is announced to all administrators.
|
||||
|
||||
@@ -104,7 +109,9 @@ What you should do on the location (strongly recommended):
|
||||
- **Files and folders** of a user: in the web interface or `occ backup:restore`, into a new folder
|
||||
("Restored <date>"), merged into the original place, or replacing it.
|
||||
- **Raw restore** of any path (for example `config/`) to a local directory: `occ backup:restore ... --to-directory=DIR`.
|
||||
- **Whole server** (disaster recovery onto a fresh installation): planned for phase 2.
|
||||
- **Whole server** (disaster recovery onto a fresh installation): `occ backup:restore:full`, see the how-to.
|
||||
- **One user** (account, settings, files, calendars, contacts via user_migration): `occ backup:user:backup` and
|
||||
`occ backup:user:restore`, replacing the account or restoring it as `<uid>-bak`. See the how-to.
|
||||
|
||||
## Storage backend experiences
|
||||
|
||||
@@ -119,6 +126,66 @@ Contributions welcome via PR — add what you ran into with your own provider.
|
||||
verified whether this also applies to a Hetzner Storage Box **main** account (only tested with a
|
||||
sub-account so far) — if you've tried a main account, please say so in a PR.
|
||||
|
||||
- **Hetzner Storage Box over WebDAV and SMB (sub-accounts, 2026-10-07):** both work end to end
|
||||
(backup, incremental run, restore of a folder, verify), tested from outside the Hetzner network
|
||||
with about 66 MiB of data. In the Hetzner Console, enable WebDAV or SMB support for the
|
||||
sub-account, and "External reachability" when your server is not in the Hetzner network.
|
||||
WebDAV: host `https://uXXXXX-subN.your-storagebox.de`, and the `root`
|
||||
must be a folder that already exists (or empty): WebDAV does not create missing parent folders,
|
||||
so a root that doesn't exist yet fails `backup:target:add` with `Sabre\HTTP\ClientHttpException:
|
||||
Conflict`. SMB: share = the sub-account name, port 445 must be reachable from your server (it was
|
||||
from a home connection here, but some providers block it), and the `smbclient` PECL extension is
|
||||
required (see the SMB entry below). Both have a noticeable fixed cost per run: an incremental run
|
||||
that uploads nothing took about 20 s over WebDAV and 50 s over SMB. A deep verify is slow over
|
||||
remote storage, since each blob is fetched with its own request (19 minutes for 66 MiB over
|
||||
WebDAV); run it occasionally, not after every backup.
|
||||
|
||||
- **FTP, server-dependent absolute-path handling:** files_external's FTP backend always builds
|
||||
absolute paths (e.g. `MKD /repo`) for the configured root, never relative ones. `atmoz/sftp`-style
|
||||
`vsftpd` test servers (chrooted) can reject that with a generic "create directory operation
|
||||
failed", even though a plain `ftp_mkdir()` with a *relative* path on the same server succeeds;
|
||||
`pure-ftpd` accepts the absolute form fine. If a working FTP server rejects ng_backup's writes,
|
||||
try a different FTP server implementation before assuming the backend itself is broken.
|
||||
|
||||
- **SMB/CIFS *lists* as available with just the `smbclient` CLI binary, but does not actually
|
||||
work for ng_backup without the `smbclient` PECL extension too.** files_external falls back to a
|
||||
shell-wrapped `smbclient` process when the native extension is missing, which registers the
|
||||
backend in `occ backup:target:backends` fine, but that fallback's stream cannot seek. ng_backup
|
||||
reads blobs back by byte range while backing up, so even the very first (small) backup fails
|
||||
with `Seek failed in packs/...`. Install the extension (`libsmbclient-dev` + `pecl install
|
||||
smbclient` + `docker-php-ext-enable smbclient` on Debian-based images), not just the CLI tool —
|
||||
the backend being listed is not proof it will work here. Once the extension is present it works
|
||||
the same as any other backend; no ng_backup-specific config beyond host/share/user/password.
|
||||
|
||||
- **NFS is not a files_external backend at all.** Mount the NFS export on the server yourself
|
||||
(e.g. via `/etc/fstab` or a systemd `.mount` unit) and point ng_backup's `local` backend at the
|
||||
mount point; ng_backup never mounts or unmounts anything on its own, so an NFS mount dropping
|
||||
mid-run behaves like any other local-path failure (the run resumes). See PLAN.md 3.5 for why NFS
|
||||
can't be driven from PHP directly.
|
||||
|
||||
## Known limitations
|
||||
|
||||
On the backlog; none of these block the beta or 1.0.0, but know them before you rely on it:
|
||||
|
||||
- **Very large instances** (well over 1 TB, millions of files): the blob index, cleanup, browsing a
|
||||
snapshot in the web interface and per-user export manifests are held in memory. Expect high
|
||||
memory use there; reports from such instances are very welcome.
|
||||
- **One location, one server.** Never let two Nextcloud servers write to the same location (for
|
||||
example a disaster recovery test server pointed at the production location): they can break each
|
||||
other's catalog, and the location is then refused as possibly tampered with. Test a disaster
|
||||
recovery against a copy of the location, or only read from it.
|
||||
- **Removing a passphrase slot does not revoke access.** The master key stays the same, so someone
|
||||
with an old recovery kit and that slot's passphrase can still decrypt the location if they can
|
||||
read it. Key rotation is not available yet.
|
||||
- **Rollback before a disaster recovery is not detected.** A fresh server has no record of the
|
||||
newest catalog generation, so a location rolled back to an older state looks valid there.
|
||||
- Location passwords passed with `-o password=...` are visible in the process list and shell
|
||||
history while the command runs.
|
||||
- Temporary files of a restore into a user's files end up in that user's trash.
|
||||
- Restoring the database does not order tables by foreign keys; apps with cascading foreign keys
|
||||
could lose rows (Nextcloud core does not use them).
|
||||
- A pack whose index upload failed stays on the location until removed by hand.
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Purpose |
|
||||
@@ -129,3 +196,7 @@ Contributions welcome via PR — add what you ran into with your own provider.
|
||||
| `backup:run`, `backup:status`, `backup:list` | back up, status, restore points |
|
||||
| `backup:browse`, `backup:restore` | browse and restore |
|
||||
| `backup:retention`, `backup:prune`, `backup:trash`, `backup:trash:restore` | retention, cleanup, trash |
|
||||
| `backup:verify` | checksum audit of a restore point (`--deep` downloads and decrypts everything) |
|
||||
| `backup:restore:full` | whole-server restore onto a fresh installation (after `backup:key:import`) |
|
||||
| `backup:key:import` | import a recovery kit on a fresh installation |
|
||||
| `backup:user:backup`, `backup:user:restore` | one user via user_migration (asks before installing it) |
|
||||
|
||||
+19
-6
@@ -4,17 +4,25 @@
|
||||
<id>ng_backup</id>
|
||||
<name>NG Backup</name>
|
||||
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
||||
<description><![CDATA[Alpha: feature-complete for scheduled, encrypted, incremental backups to
|
||||
S3/SFTP/WebDAV/local, with restore of files and folders from the web interface and occ. Meant for
|
||||
testing on a copy of production (dev, test, staging) -- not yet for production use, and full
|
||||
disaster recovery is not implemented yet. See the recovery kit section of the README before
|
||||
relying on this for anything.]]></description>
|
||||
<version>0.1.0</version>
|
||||
<description><![CDATA[Beta: scheduled, encrypted, incremental backups of the database, config and data
|
||||
directory to S3, SFTP, WebDAV, FTP, SMB or a local path, with restore of files and folders, of
|
||||
single users (via user_migration) and of the whole server onto a fresh installation. Tested
|
||||
thoroughly and usable alongside your current backup, but not yet a drop-in replacement for it.
|
||||
See the README for known limitations and the recovery kit before relying on it.]]></description>
|
||||
<version>0.9.3</version>
|
||||
<licence>agpl</licence>
|
||||
<author>Dillard Blom</author>
|
||||
<namespace>NgBackup</namespace>
|
||||
<documentation>
|
||||
<admin>https://github.com/dillardblom/ng_backup/blob/main/docs/HOWTO.md</admin>
|
||||
</documentation>
|
||||
<category>tools</category>
|
||||
<website>https://apps.nextcloud.com/apps/ng_backup</website>
|
||||
<bugs>https://github.com/dillardblom/ng_backup/issues</bugs>
|
||||
<repository type="git">https://github.com/dillardblom/ng_backup</repository>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/1-backups.png</screenshot>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/2-restore.png</screenshot>
|
||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/3-settings.png</screenshot>
|
||||
<dependencies>
|
||||
<php min-version="8.1"/>
|
||||
<lib>sodium</lib>
|
||||
@@ -47,6 +55,11 @@ relying on this for anything.]]></description>
|
||||
<command>OCA\NgBackup\Command\KeySlotAdd</command>
|
||||
<command>OCA\NgBackup\Command\KeySlotReplace</command>
|
||||
<command>OCA\NgBackup\Command\KeySlotRemove</command>
|
||||
<command>OCA\NgBackup\Command\BackupUser</command>
|
||||
<command>OCA\NgBackup\Command\RestoreUser</command>
|
||||
<command>OCA\NgBackup\Command\KeyImport</command>
|
||||
<command>OCA\NgBackup\Command\RestoreFull</command>
|
||||
<command>OCA\NgBackup\Command\Verify</command>
|
||||
</commands>
|
||||
<settings>
|
||||
<admin>OCA\NgBackup\Settings\Admin</admin>
|
||||
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
# NG Backup how-to
|
||||
|
||||
Step by step: from first install to a tested restore. All commands run as the web user
|
||||
(`sudo -u www-data php occ ...`, or `docker exec -u www-data <container> php occ ...` in
|
||||
All-in-One and Docker setups). Replace names in `<angle brackets>` with your own.
|
||||
|
||||
## 1. Create the key and the recovery kit
|
||||
|
||||
```
|
||||
occ backup:key:init --label="Main" --delete-delay-days=7
|
||||
occ backup:key:kit --output=/var/tmp/ng_backup-kit.json
|
||||
occ backup:key:confirm -u <your-admin-uid> --code=<code shown> --confirm="Yes, I confirm"
|
||||
```
|
||||
|
||||
- The passphrase protects the backup. Keep it somewhere other than this server.
|
||||
- Optional: extra passphrases for other people with `occ backup:key:slot:add --label="..."`.
|
||||
Any one passphrase is enough to restore. Removing a slot later does not revoke access for someone
|
||||
who kept an old kit and that passphrase (see "Known limitations" in the README).
|
||||
- The recovery kit (`/var/tmp/ng_backup-kit.json` above) is also required for disaster recovery. The web
|
||||
user writes it there; then move it off the server as root, next to the passphrase, and delete the
|
||||
copy in `/var/tmp`. Without the kit and a passphrase, the backup cannot be read.
|
||||
|
||||
## 2. Add a backup location
|
||||
|
||||
NG Backup uses Nextcloud's External storage backends. Check what your server supports first:
|
||||
|
||||
```
|
||||
occ backup:target:backends
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
occ backup:target:add offsite amazons3 -a amazons3::accesskey -o bucket=<bucket> -o key=<key> -o secret=<secret>
|
||||
occ backup:target:add nas sftp -a password::password -o host=<host> -o user=<user> -o password=<pw> -o root=home
|
||||
occ backup:target:add local local -a null::null -o datadir=/mnt/backup
|
||||
```
|
||||
|
||||
- **SFTP on a Hetzner Storage Box sub-account:** use `-o root=home`, not `/` (see README).
|
||||
- **FTP:** use a server that accepts absolute paths (for example `pure-ftpd`).
|
||||
- **SMB:** requires the native `smbclient` PHP extension. The CLI binary alone is not enough (see README).
|
||||
`backup:target:add` refuses SMB when the extension is missing.
|
||||
- **NFS:** not a backend of its own. Mount the export on the server and use the `local` backend
|
||||
with the mount point as `datadir`.
|
||||
|
||||
Verify the location before relying on it:
|
||||
|
||||
```
|
||||
occ backup:target:list
|
||||
occ backup:status
|
||||
```
|
||||
|
||||
## 3. Run the first backup
|
||||
|
||||
```
|
||||
occ backup:run offsite -l first
|
||||
occ backup:list
|
||||
```
|
||||
|
||||
Later runs upload only changed data. An unchanged run uploads (almost) nothing; the output shows
|
||||
the number of uploaded bytes, so check that value when you test incremental behaviour.
|
||||
|
||||
Schedule `occ backup:run` from system cron (or use the Administration settings, if you prefer).
|
||||
|
||||
## 4. Check the backup regularly
|
||||
|
||||
```
|
||||
occ backup:verify offsite <snapshot> # shallow: all blocks are present
|
||||
occ backup:verify offsite <snapshot> --deep # also downloads and decrypts every block
|
||||
```
|
||||
|
||||
Run the deep check at least once after setting up a location, and after changing it.
|
||||
|
||||
## 5. Restore files and folders
|
||||
|
||||
In the web interface, or with occ:
|
||||
|
||||
```
|
||||
occ backup:browse offsite <snapshot>
|
||||
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=new-folder
|
||||
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=merge
|
||||
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=replace
|
||||
occ backup:restore offsite <snapshot> config --to-directory=/var/tmp/restore-config
|
||||
```
|
||||
|
||||
- `new-folder` (default): puts the files in "Restored <date>" next to the original. Safest choice.
|
||||
- `merge`: adds missing files, keeps current ones.
|
||||
- `replace`: overwrites the folder. Replaced files keep their old content as a version, and removed
|
||||
files go to the trash.
|
||||
- `--to-directory`: writes raw files to a local directory (for example `config/`).
|
||||
|
||||
## 6. Back up and restore one user
|
||||
|
||||
Needs the `user_migration` app. NG Backup asks before installing it:
|
||||
|
||||
```
|
||||
occ backup:user:backup offsite <uid>
|
||||
occ backup:user:restore offsite users/<uid>/<exportId> # replace (default)
|
||||
occ backup:user:restore offsite users/<uid>/<exportId> --mode=as-backup
|
||||
```
|
||||
|
||||
- If `user_migration` is missing, the command asks to install and enable it. For non-interactive
|
||||
use, add `--install-user-migration` to approve that explicitly. The command stops after the
|
||||
installation; run it once more.
|
||||
- `replace` first makes a safety export of the current account, then recreates the user under the
|
||||
same id. When the account exists, the command asks first; in scripts, add `--force`. Shares to
|
||||
others and data of apps without a user_migration migrator are not restored, so prefer
|
||||
`as-backup` when in doubt: it restores into `<uid>-bak` and leaves the original untouched.
|
||||
- A restore onto a server where the user does not exist works. Calendars and address books get an
|
||||
internal name with the prefix `migrated-` (for example `migrated-personal`). That prefix comes from
|
||||
user_migration and cannot be turned off. The user can change the display name of these calendars and
|
||||
address books in the app as usual; the internal name stays as it is.
|
||||
- **Settings:** on restore, `user_migration` imports only the app settings on its own allowlist
|
||||
(for example calendar view and reminder settings). Other app settings of the user are skipped.
|
||||
NG Backup reuses `user_migration`'s importer, so this list is not extended.
|
||||
- Keep the last few exports per user: `occ backup:retention --user-last=3`.
|
||||
|
||||
## 7. Disaster recovery: restore the whole server
|
||||
|
||||
Use a fresh Nextcloud installation of the same major version, with NG Backup installed and
|
||||
the same location available. For a test, use a **copy** of the location: two servers writing to one
|
||||
location break each other's catalog.
|
||||
|
||||
```
|
||||
occ backup:key:import /path/ng_backup-kit.json --passphrase-file=/path/passphrase
|
||||
occ backup:target:add offsite <backend> ... # same details as on the original server
|
||||
occ backup:list offsite # the snapshots must be listed
|
||||
occ backup:restore:full offsite <snapshot>
|
||||
```
|
||||
|
||||
The command refuses on an installation that already has more than one user, and asks before it
|
||||
starts; `--force` skips both (also needed to rerun it after it stopped halfway).
|
||||
|
||||
Afterwards, check that users, files and the database are complete, and run a backup of the restored
|
||||
server. The restored installation keeps its own app configuration and secret.
|
||||
|
||||
## 8. Retention
|
||||
|
||||
```
|
||||
occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12
|
||||
occ backup:prune offsite --dry-run # show first
|
||||
occ backup:prune offsite # apply the policy; forgotten snapshots go to the trash first
|
||||
occ backup:trash # list the trash
|
||||
occ backup:trash:restore offsite <id>
|
||||
```
|
||||
|
||||
Snapshots are kept according to the policy, and their data is deleted only after the deletion delay
|
||||
(set with `backup:key:init --delete-delay-days`). `occ backup:prune offsite --dry-run` shows what would be
|
||||
removed without changing anything.
|
||||
|
||||
## 9. Checklist before you rely on it
|
||||
|
||||
- [ ] Key created, passphrase stored off-server, recovery kit stored off-server
|
||||
- [ ] Location added; `backup:status` shows it reachable
|
||||
- [ ] First backup done and listed in `backup:list`
|
||||
- [ ] `backup:verify --deep` succeeds
|
||||
- [ ] One file restore tested (`new-folder`)
|
||||
- [ ] Disaster recovery tested on a second, fresh installation (or in a test environment)
|
||||
@@ -37,5 +37,15 @@ interface IBackend {
|
||||
/** @return list<string> object paths below $prefix (recursive) */
|
||||
public function list(string $prefix): array;
|
||||
|
||||
/**
|
||||
* Object paths directly in $dir, for a folder that only holds objects (index/, snapshots/,
|
||||
* catalog/, trash/info/). One listing, without checking the type of every entry: on a remote
|
||||
* storage list() costs a round trip per entry, which for index/ (one file per pack) grows
|
||||
* with the size of the repository.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
public function listFiles(string $dir): array;
|
||||
|
||||
public function delete(string $path): void;
|
||||
}
|
||||
|
||||
@@ -33,11 +33,16 @@ final class LocalBackend implements IBackend {
|
||||
if ($out === false) {
|
||||
throw new BackendException('Cannot write ' . $path);
|
||||
}
|
||||
$expected = StorageBackend::remaining($stream);
|
||||
if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) {
|
||||
throw new BackendException('Write failed for ' . $path);
|
||||
}
|
||||
fclose($out);
|
||||
$out = false;
|
||||
clearstatcache(true, $tmp);
|
||||
if ($expected !== null && filesize($tmp) !== $expected) {
|
||||
throw new BackendException('Incomplete write for ' . $path);
|
||||
}
|
||||
if (!rename($tmp, $target)) {
|
||||
throw new BackendException('Cannot finalise ' . $path);
|
||||
}
|
||||
@@ -94,7 +99,7 @@ final class LocalBackend implements IBackend {
|
||||
$result = [];
|
||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS));
|
||||
foreach ($it as $file) {
|
||||
if ($file->isFile() && !str_contains($file->getFilename(), '.part-')) {
|
||||
if ($file->isFile() && !StorageBackend::isTemporary($file->getFilename())) {
|
||||
$result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/');
|
||||
}
|
||||
}
|
||||
@@ -102,6 +107,25 @@ final class LocalBackend implements IBackend {
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function listFiles(string $dir): array {
|
||||
$base = $this->abs($dir);
|
||||
if (!is_dir($base)) {
|
||||
return [];
|
||||
}
|
||||
$result = [];
|
||||
$names = scandir($base);
|
||||
if ($names === false) {
|
||||
throw new BackendException('Cannot list ' . $dir);
|
||||
}
|
||||
foreach ($names as $name) {
|
||||
if ($name !== '.' && $name !== '..' && !StorageBackend::isTemporary($name)) {
|
||||
$result[] = trim($dir, '/') . '/' . $name;
|
||||
}
|
||||
}
|
||||
sort($result);
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function delete(string $path): void {
|
||||
@unlink($this->abs($path));
|
||||
}
|
||||
|
||||
@@ -30,7 +30,13 @@ final class StorageBackend implements IBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/** A temporary upload of put(): "<name>.part-<8 hex>". */
|
||||
public static function isTemporary(string $name): bool {
|
||||
return preg_match('/\.part-[0-9a-f]{8}$/', $name) === 1;
|
||||
}
|
||||
|
||||
public function put(string $path, $stream): void {
|
||||
$expected = self::remaining($stream);
|
||||
$target = $this->abs($path);
|
||||
$this->mkdirs(dirname($target));
|
||||
$writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4));
|
||||
@@ -54,6 +60,10 @@ final class StorageBackend implements IBackend {
|
||||
if (is_resource($stream)) {
|
||||
fclose($stream);
|
||||
}
|
||||
// Some adapters report success after a short write or a failed flush: check the size.
|
||||
if ($expected !== null && $this->storage->filesize($writeTo) !== $expected) {
|
||||
throw new BackendException('Incomplete write for ' . $path);
|
||||
}
|
||||
if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) {
|
||||
throw new BackendException('Cannot finalise ' . $path);
|
||||
}
|
||||
@@ -123,6 +133,28 @@ final class StorageBackend implements IBackend {
|
||||
}
|
||||
}
|
||||
|
||||
public function listFiles(string $dir): array {
|
||||
$abs = $this->abs($dir);
|
||||
if (!$this->storage->is_dir($abs)) {
|
||||
return []; // not created yet
|
||||
}
|
||||
$dh = $this->storage->opendir($abs);
|
||||
if ($dh === false) {
|
||||
// An existing folder read as empty would look like a new repository.
|
||||
throw new BackendException('Cannot list ' . $dir);
|
||||
}
|
||||
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
||||
$result = [];
|
||||
while (($name = readdir($dh)) !== false) {
|
||||
if ($name !== '.' && $name !== '..' && !self::isTemporary($name)) {
|
||||
$result[] = substr($abs . '/' . $name, $cut);
|
||||
}
|
||||
}
|
||||
closedir($dh);
|
||||
sort($result);
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function listInto(string $dir, array &$result): void {
|
||||
if (!$this->storage->is_dir($dir)) {
|
||||
return;
|
||||
@@ -132,19 +164,39 @@ final class StorageBackend implements IBackend {
|
||||
return;
|
||||
}
|
||||
while (($name = readdir($dh)) !== false) {
|
||||
if ($name === '.' || $name === '..' || str_contains($name, '.part-')) {
|
||||
if ($name === '.' || $name === '..') {
|
||||
continue;
|
||||
}
|
||||
$child = $dir . '/' . $name;
|
||||
if ($this->storage->is_dir($child)) {
|
||||
$this->listInto($child, $result);
|
||||
} else {
|
||||
} elseif (!self::isTemporary($name)) {
|
||||
$result[] = $child;
|
||||
}
|
||||
}
|
||||
closedir($dh);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bytes left to read in a local $stream (php://temp, php://memory or a plain file), or null:
|
||||
* stream wrappers of remote storages do not report a reliable size.
|
||||
*/
|
||||
public static function remaining(mixed $stream): ?int {
|
||||
if (!is_resource($stream)) {
|
||||
return null;
|
||||
}
|
||||
$meta = stream_get_meta_data($stream);
|
||||
if (!($meta['seekable'] ?? false) || !in_array($meta['wrapper_type'] ?? '', ['PHP', 'plainfile'], true)) {
|
||||
return null;
|
||||
}
|
||||
$stat = fstat($stream);
|
||||
$pos = ftell($stream);
|
||||
if ($stat === false || $pos === false) {
|
||||
return null;
|
||||
}
|
||||
return max(0, $stat['size'] - $pos);
|
||||
}
|
||||
|
||||
private function mkdirs(string $dir): void {
|
||||
if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) {
|
||||
return;
|
||||
|
||||
@@ -38,13 +38,19 @@ final class BackupJob extends TimedJob {
|
||||
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
|
||||
}
|
||||
|
||||
private const MAX_DAILY_ATTEMPTS = 3;
|
||||
|
||||
protected function run($argument): void {
|
||||
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
|
||||
$deadline = microtime(true) + $budget;
|
||||
$this->startScheduled();
|
||||
foreach ($this->runs->findRunning(null, BackupService::KIND_FULL) as $run) {
|
||||
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
|
||||
$run = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
||||
$next = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
||||
if ($next === $run) {
|
||||
break; // step() hands back the same object when another process holds the run's lease
|
||||
}
|
||||
$run = $next;
|
||||
}
|
||||
}
|
||||
$this->failStaleRestores();
|
||||
@@ -67,10 +73,15 @@ final class BackupJob extends TimedJob {
|
||||
}
|
||||
}
|
||||
|
||||
/** Once a week (after Sunday's backup), apply the retention policy to non-append-only locations. */
|
||||
/**
|
||||
* Once a week, apply the retention policy to non-append-only locations: from Sunday on, at the
|
||||
* first moment no backup is running, so a long Sunday backup only delays it instead of skipping
|
||||
* the week.
|
||||
*/
|
||||
private function weeklyPrune(): void {
|
||||
$lastWeek = $this->appConfig->getValueString(Application::APP_ID, 'last_prune_week', '');
|
||||
if (date('N') !== '7' || $lastWeek === date('o-W') || $this->runs->findRunning(null, BackupService::KIND_FULL) !== []) {
|
||||
$due = date('N') === '7' ? date('o-W') : date('o-W', (int)strtotime('last sunday'));
|
||||
if (strcmp($lastWeek, $due) >= 0 || $this->runs->findRunning(null, BackupService::KIND_FULL) !== []) {
|
||||
return;
|
||||
}
|
||||
foreach ($this->targets->list() as $t) {
|
||||
@@ -83,7 +94,7 @@ final class BackupJob extends TimedJob {
|
||||
$this->logger->warning('NG Backup: cleanup of ' . $t->getName() . ' failed: ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', date('o-W'));
|
||||
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', $due);
|
||||
}
|
||||
|
||||
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
|
||||
@@ -92,7 +103,7 @@ final class BackupJob extends TimedJob {
|
||||
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
|
||||
return;
|
||||
}
|
||||
$todayAt = strtotime(date('Y-m-d') . ' ' . $at);
|
||||
$todayAt = (int)strtotime(date('Y-m-d') . ' ' . $at);
|
||||
if (time() < $todayAt) {
|
||||
return;
|
||||
}
|
||||
@@ -101,6 +112,12 @@ final class BackupJob extends TimedJob {
|
||||
if ($this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
|
||||
continue;
|
||||
}
|
||||
// After a failure: retry at most MAX_DAILY_ATTEMPTS times a day, an hour apart, instead
|
||||
// of starting (and notifying about) a new full run every five minutes all day.
|
||||
$today = $this->runs->startedSince($t->getId(), BackupService::KIND_FULL, $todayAt);
|
||||
if (count($today) >= self::MAX_DAILY_ATTEMPTS || ($today !== [] && time() - (int)$today[0]->getFinishedAt() < 3600)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
$this->backups->start($t, 'scheduled');
|
||||
} catch (\Throwable $e) {
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use OCA\NgBackup\Service\UserBackupService;
|
||||
use OCA\NgBackup\UserMigration\UserMigrationInstaller;
|
||||
use OCP\IUserManager;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class BackupUser extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private UserBackupService $userBackup,
|
||||
private IUserManager $userManager,
|
||||
private UserMigrationInstaller $installer,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:user:backup')
|
||||
->setDescription('Back up one user via user_migration (account, settings, files and everything else a migrator covers)')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('user', InputArgument::REQUIRED, 'User id')
|
||||
->addOption('install-user-migration', null, InputOption::VALUE_NONE, 'Install and enable user_migration if it is missing (non-interactive consent)')
|
||||
->setHelp(<<<'HELP'
|
||||
Streams a user_migration export straight into the encrypted repository: no zip, no copy in the
|
||||
user's own storage. Shares data with the regular backup; this needs the user_migration app
|
||||
enabled. The manifest path this prints is what occ backup:user:restore expects.
|
||||
|
||||
Example:
|
||||
occ backup:user:backup offsite alice
|
||||
HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$uid = (string)$input->getArgument('user');
|
||||
$user = $this->userManager->get($uid);
|
||||
if ($user === null) {
|
||||
$output->writeln("<error>No such user: $uid</error>");
|
||||
return 1;
|
||||
}
|
||||
try {
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$this->installer->ensureEnabled($input, $output);
|
||||
$manifest = $this->userBackup->backupUser($target, $user);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln("Backed up <info>$uid</info> to $manifest.");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\KeyService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Question\Question;
|
||||
|
||||
final class KeyImport extends Command {
|
||||
public function __construct(
|
||||
private KeyService $keys,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:key:import')
|
||||
->setDescription('Disaster recovery: adopt the backup key from a recovery kit on a fresh installation')
|
||||
->addArgument('kit-file', InputArgument::REQUIRED, 'The recovery kit file (occ backup:key:kit on the original installation)')
|
||||
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking')
|
||||
->setHelp(<<<'HELP'
|
||||
For a fresh installation only (occ backup:key:init must not have been run here yet). Reads one
|
||||
of the kit's passphrase slots to recover the original master key, so occ backup:target:add opens
|
||||
the existing repository on the location instead of creating a new, empty one.
|
||||
|
||||
Example:
|
||||
occ backup:key:import /root/ng_backup-recovery-kit.json
|
||||
occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=...
|
||||
occ backup:restore:full offsite <snapshot>
|
||||
HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
if ($this->keys->isInitialized()) {
|
||||
$output->writeln('<error>A backup key already exists here (fingerprint ' . $this->keys->fingerprint() . '). occ backup:key:import is only for a fresh installation.</error>');
|
||||
return 1;
|
||||
}
|
||||
$kitFile = (string)$input->getArgument('kit-file');
|
||||
$raw = @file_get_contents($kitFile);
|
||||
if ($raw === false) {
|
||||
$output->writeln("<error>Cannot read $kitFile</error>");
|
||||
return 1;
|
||||
}
|
||||
try {
|
||||
$kit = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
|
||||
} catch (\JsonException $e) {
|
||||
$output->writeln('<error>Not a valid recovery kit file: ' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$file = $input->getOption('passphrase-file');
|
||||
if ($file !== null) {
|
||||
$raw = @file_get_contents($file);
|
||||
if ($raw === false) {
|
||||
$output->writeln('<error>Cannot read the passphrase file</error>');
|
||||
return 1;
|
||||
}
|
||||
$pass = rtrim($raw, "\r\n");
|
||||
} else {
|
||||
$helper = $this->getHelper('question');
|
||||
$q = (new Question('Passphrase of one of the kit\'s slots: '))->setHidden(true)->setHiddenFallback(false);
|
||||
$pass = (string)$helper->ask($input, $output, $q);
|
||||
}
|
||||
try {
|
||||
$this->keys->importRecoveryKit($kit, $pass);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln('Backup key imported, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
||||
// Read from the kit file, which is not authenticated beyond the wrapped key: show it so a
|
||||
// wrong or altered value stands out.
|
||||
$output->writeln('Deletion delay for forgotten snapshots: <info>' . $this->keys->deleteDelayDays() . '</info> days (from the kit).');
|
||||
$output->writeln('Next: connect the backup location with the same details as before (<info>occ backup:target:add</info>), then <info>occ backup:list</info> and <info>occ backup:restore:full</info>.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
+15
-2
@@ -39,11 +39,24 @@ final class KeyKit extends Command {
|
||||
$output->write($json, false, OutputInterface::OUTPUT_RAW);
|
||||
return 0;
|
||||
}
|
||||
if (file_put_contents($file, $json) === false) {
|
||||
// Created owner-only from the start (no window where it is world-readable), never over an
|
||||
// existing file or symlink someone else may have placed at a predictable path.
|
||||
$old = umask(0077);
|
||||
try {
|
||||
$fh = @fopen($file, 'xb');
|
||||
} finally {
|
||||
umask($old);
|
||||
}
|
||||
if ($fh === false) {
|
||||
$output->writeln("<error>Cannot create $file (it must not exist yet)</error>");
|
||||
return 1;
|
||||
}
|
||||
$ok = fwrite($fh, $json) === strlen($json);
|
||||
if (!fclose($fh) || !$ok) {
|
||||
@unlink($file);
|
||||
$output->writeln("<error>Cannot write $file</error>");
|
||||
return 1;
|
||||
}
|
||||
@chmod($file, 0600);
|
||||
$output->writeln("Recovery kit written to <info>$file</info>. Copy it off this server, then run <info>occ backup:key:confirm</info>.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -37,6 +37,22 @@ final class Prune extends Command {
|
||||
$output->writeln('Policy: ' . json_encode($this->prune->policy()->toArray()) . ($dry ? ' (dry run)' : ''));
|
||||
$rc = 0;
|
||||
foreach ($targets as $t) {
|
||||
// User exports first: Repository::prune()'s blob garbage collection (run as part of
|
||||
// apply() below) scans the users/ manifests still on the location at that point, so
|
||||
// forgetting one here lets its now-unused blobs be freed in this same pass.
|
||||
try {
|
||||
$ru = $this->prune->applyUserExports($t, $dry);
|
||||
} catch (\Throwable $e) {
|
||||
// Snapshot retention is independent of this: report and carry on with it.
|
||||
$output->writeln('<error>' . $t->getName() . ' (user exports): ' . $e->getMessage() . '</error>');
|
||||
$rc = 1;
|
||||
$ru = ['forgotten' => []];
|
||||
}
|
||||
if ($ru['forgotten'] !== []) {
|
||||
$affected = count(array_unique(array_map(static fn (string $path): string => explode('/', $path)[1] ?? '', $ru['forgotten'])));
|
||||
$output->writeln(sprintf('<info>%s</info>: %s %d user export(s) for %d user(s)',
|
||||
$t->getName(), $dry ? 'would forget' : 'forgot', count($ru['forgotten']), $affected));
|
||||
}
|
||||
try {
|
||||
$r = $this->prune->apply($t, $dry);
|
||||
} catch (\Throwable $e) {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\RestoreFullService;
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use OCP\IUserManager;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\QuestionHelper;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class RestoreFull extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private RestoreFullService $restoreFull,
|
||||
private IUserManager $userManager,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:restore:full')
|
||||
->setDescription('Disaster recovery: restore the database, data directory and config on a fresh installation')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
||||
->addOption('force', null, InputOption::VALUE_NONE, 'Also on an installation that already has more than one user, and without asking')
|
||||
->setHelp(<<<'HELP'
|
||||
For a FRESH Nextcloud installation only, same major version as the backup, right after
|
||||
occ maintenance:install, occ backup:key:import and occ backup:target:add -- before anyone else
|
||||
uses this instance.
|
||||
|
||||
Every app that was enabled when the snapshot was taken must already be installed here (its
|
||||
database tables must exist before the restore can write into them); the command lists what is
|
||||
missing and refuses to continue rather than guess.
|
||||
|
||||
Steps: the database is replaced (consistent, in one transaction); the data directory is restored
|
||||
directly in place; config.php is merged in -- everything from the backup except the database
|
||||
connection and data directory settings, which stay as occ maintenance:install configured them
|
||||
here. Other files under config/ (if any) are not restored.
|
||||
|
||||
Afterwards: check trusted_domains and overwritehost/overwriteprotocol match this server, and
|
||||
occ maintenance:repair.
|
||||
|
||||
Example:
|
||||
occ backup:restore:full offsite 537bf7e9c1a4...
|
||||
HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$force = (bool)$input->getOption('force');
|
||||
// A whole-server restore replaces the database and data directory of THIS instance. On
|
||||
// anything but a fresh install (only the admin from occ maintenance:install) that is
|
||||
// almost certainly a mistake, so it needs --force.
|
||||
$users = $this->userManager->countUsersTotal(2);
|
||||
if (!$force && ($users === false || $users > 1)) {
|
||||
$output->writeln('<error>This installation already has more than one user: backup:restore:full would replace its whole database and data directory. It is meant for a fresh installation. Use --force if this really is what you want.</error>');
|
||||
return 1;
|
||||
}
|
||||
if (!$force && $input->isInteractive()) {
|
||||
$q = new ConfirmationQuestion('This replaces the database, the data directory and config.php of this installation. Continue? [y/N] ', false);
|
||||
if (!(new QuestionHelper())->ask($input, $output, $q)) {
|
||||
$output->writeln('Cancelled.');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
try {
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$snapshot = (string)$input->getArgument('snapshot');
|
||||
$result = $this->restoreFull->restoreFull($target, $snapshot);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
$output->writeln('If this happened after the restore had started, the instance may be partly restored: fix the cause and run the same command again with --force.');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln(sprintf(
|
||||
'Database restored (<info>%d</info> tables, <info>%d</info> unchanged), config merged (<info>%d</info> keys), <info>%d</info> data files restored.',
|
||||
count($result['dbReport']['restored']),
|
||||
count($result['dbReport']['skipped']),
|
||||
$result['configKeys'],
|
||||
$result['dataFiles'],
|
||||
));
|
||||
if (!$result['dbConsistent']) {
|
||||
$output->writeln('<comment>Warning: this snapshot\'s database dump was made over several steps (web cron), so it is not one consistent point in time. Check the instance carefully; prefer a snapshot made with system cron.</comment>');
|
||||
}
|
||||
$output->writeln('Next: check trusted_domains and overwritehost/overwriteprotocol match this server, then occ maintenance:repair.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use OCA\NgBackup\Service\UserRestoreService;
|
||||
use OCA\NgBackup\UserMigration\UserMigrationInstaller;
|
||||
use OCP\IUserManager;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Helper\QuestionHelper;
|
||||
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class RestoreUser extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private UserRestoreService $userRestore,
|
||||
private UserMigrationInstaller $installer,
|
||||
private IUserManager $userManager,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:user:restore')
|
||||
->setDescription('Restore a user from a user_migration export (see backup:user:backup)')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('manifest', InputArgument::REQUIRED, 'Manifest path printed by backup:user:backup (users/<uid>/<exportId>)')
|
||||
->addOption('mode', 'm', InputOption::VALUE_REQUIRED, 'replace or as-backup', UserRestoreService::MODE_REPLACE)
|
||||
->addOption('force', null, InputOption::VALUE_NONE, 'Replace an existing account without asking (required when not interactive)')
|
||||
->addOption('install-user-migration', null, InputOption::VALUE_NONE, 'Install and enable user_migration if it is missing (non-interactive consent)')
|
||||
->setHelp(<<<'HELP'
|
||||
Never imports over a live account. Modes:
|
||||
replace (default) safety-exports the current account first, then deletes and recreates it
|
||||
fresh from the backup, under the same user id
|
||||
as-backup restores into a new account "<uid>-bak" (or "-bak-2", ...) instead, so an admin can
|
||||
compare and transfer by hand; Nextcloud cannot rename a user id, so the backup gets
|
||||
the new name, never the existing account
|
||||
|
||||
Example:
|
||||
occ backup:user:restore offsite users/alice/20261005T120000Z-ab12cd34
|
||||
occ backup:user:restore offsite users/alice/20261005T120000Z-ab12cd34 --mode=as-backup
|
||||
HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$mode = (string)$input->getOption('mode');
|
||||
$uid = explode('/', (string)$input->getArgument('manifest'))[1] ?? '';
|
||||
if ($mode === UserRestoreService::MODE_REPLACE && $uid !== '' && $this->userManager->userExists($uid) && !$input->getOption('force')) {
|
||||
// Replace deletes the live account (after a safety export). Shares to others and data of
|
||||
// apps without a user_migration migrator do not come back, so never do it unasked.
|
||||
if (!$input->isInteractive()) {
|
||||
$output->writeln("<error>$uid exists: --mode=replace deletes and recreates it. Add --force, or use --mode=as-backup.</error>");
|
||||
return 1;
|
||||
}
|
||||
$q = new ConfirmationQuestion("$uid exists and will be deleted and recreated from the backup (a safety export is made first; its shares to others and data of apps without a migrator are not restored). Continue? [y/N] ", false);
|
||||
if (!(new QuestionHelper())->ask($input, $output, $q)) {
|
||||
$output->writeln('Cancelled.');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
try {
|
||||
$output->writeln('<comment>Note: on restore, user_migration only imports the app settings on its own allowlist (e.g. calendar view and reminder settings); other app settings of this user are skipped. Reason: NG Backup uses user_migration\'s importer; widening the list would mean building a settings importer of our own.</comment>');
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$this->installer->ensureEnabled($input, $output);
|
||||
$user = $this->userRestore->restoreUser($target, (string)$input->getArgument('manifest'), $mode);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln('Restored <info>' . $user->getUID() . '</info>.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -28,10 +28,15 @@ final class Retention extends Command {
|
||||
->addOption('daily', null, InputOption::VALUE_REQUIRED, 'Keep one per day for the last N days with a backup')
|
||||
->addOption('weekly', null, InputOption::VALUE_REQUIRED, 'Keep one per week for N weeks')
|
||||
->addOption('monthly', null, InputOption::VALUE_REQUIRED, 'Keep one per month for N months')
|
||||
->setHelp("Example: occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12");
|
||||
->addOption('user-last', null, InputOption::VALUE_REQUIRED, 'Keep the newest N of each user\'s own exports (occ backup:user:backup)')
|
||||
->setHelp("Example: occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12 --user-last=3");
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
if ($input->getOption('user-last') !== null && !ctype_digit((string)$input->getOption('user-last'))) {
|
||||
$output->writeln('<error>--user-last must be a whole number</error>');
|
||||
return 1;
|
||||
}
|
||||
$p = $this->prune->policy()->toArray();
|
||||
$changed = false;
|
||||
foreach (['last', 'daily', 'weekly', 'monthly'] as $k) {
|
||||
@@ -48,7 +53,16 @@ final class Retention extends Command {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if ($input->getOption('user-last') !== null) {
|
||||
try {
|
||||
$this->prune->setUserExportRetention((int)$input->getOption('user-last'));
|
||||
} catch (\InvalidArgumentException $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
$output->writeln(sprintf('Keep the last %d, plus one per day for %d days, per week for %d weeks, per month for %d months.', $p['last'], $p['daily'], $p['weekly'], $p['monthly']));
|
||||
$output->writeln(sprintf('Keep the newest %d of each user\'s own exports.', $this->prune->userExportRetention()));
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +75,13 @@ HELP);
|
||||
});
|
||||
if ($run->getStatus() === Run::DONE) {
|
||||
$output->writeln(sprintf(' done in %.0fs', microtime(true) - $start));
|
||||
$f = json_decode($run->getStats() ?? '{}', true)['files'] ?? [];
|
||||
if (($f['unreadable'] ?? 0) > 0) {
|
||||
$output->writeln(sprintf('<comment> warning: %d unreadable files or folders are missing from this backup (see the server log)</comment>', $f['unreadable']));
|
||||
}
|
||||
if (($f['changing'] ?? 0) > 0) {
|
||||
$output->writeln(sprintf('<comment> warning: %d files kept changing and were saved as read (see the server log)</comment>', $f['changing']));
|
||||
}
|
||||
} else {
|
||||
$output->writeln('<error> ' . $run->getStatus() . ': ' . $run->getError() . '</error>');
|
||||
$failed++;
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Command;
|
||||
|
||||
use OCA\NgBackup\Service\TargetService;
|
||||
use OCA\NgBackup\Service\VerifyService;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
final class Verify extends Command {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private VerifyService $verify,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
$this->setName('backup:verify')
|
||||
->setDescription('Checksum audit of a snapshot: every blob it needs must be in the (freshly re-checked) index and its pack present')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
||||
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
||||
->addOption('deep', null, InputOption::VALUE_NONE, 'Also download and decrypt every blob (verifies its authentication tag and content hash, not just that it is listed); much slower')
|
||||
->setHelp(<<<'HELP'
|
||||
Without --deep this re-downloads and re-authenticates the index (never trusting the local cache)
|
||||
and confirms every blob the snapshot needs is listed and its pack is present -- fast, no blob
|
||||
content is read, so it cannot by itself catch a truncated or bit-rotted pack. With --deep every
|
||||
blob is also downloaded and decrypted, which fails loudly on exactly that even if the index still
|
||||
lists it -- the same verification an actual restore would do, run here deliberately instead of
|
||||
waiting to find out during one.
|
||||
|
||||
Run this periodically (cron, picking a sample of recent snapshots) rather than only right before
|
||||
you'd need to rely on a restore.
|
||||
|
||||
Examples:
|
||||
occ backup:verify offsite 537bf7e9c1a4...
|
||||
occ backup:verify offsite 537bf7e9c1a4... --deep
|
||||
HELP);
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||
try {
|
||||
$target = $this->targets->get($input->getArgument('target'));
|
||||
$ref = (string)$input->getArgument('snapshot');
|
||||
$deep = (bool)$input->getOption('deep');
|
||||
$result = str_starts_with($ref, 'users/')
|
||||
? $this->verify->verifyUserExport($target, $ref, $deep)
|
||||
: $this->verify->verify($target, $ref, $deep);
|
||||
} catch (\Throwable $e) {
|
||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||
return 1;
|
||||
}
|
||||
$output->writeln(sprintf('%d files, %d blobs checked%s.', $result['filesChecked'], $result['blobsChecked'],
|
||||
$deep ? sprintf(' (%.1f MiB decrypted)', $result['bytesChecked'] / 1048576) : ''));
|
||||
if ($result['missing'] !== []) {
|
||||
$output->writeln('<error>' . count($result['missing']) . ' blob(s) missing from the index:</error> ' . implode(', ', $result['missing']));
|
||||
}
|
||||
if ($result['failed'] !== []) {
|
||||
$output->writeln('<error>' . count($result['failed']) . ' blob(s) failed verification (corrupted or tampered):</error> ' . implode(', ', $result['failed']));
|
||||
}
|
||||
if ($result['unreadable'] !== []) {
|
||||
$output->writeln('<error>' . count($result['unreadable']) . ' blob(s) could not be read from the location</error> (often a temporary storage '
|
||||
. 'or network error: run the check again; if the same blobs stay unreadable, their pack may be damaged or truncated): '
|
||||
. implode(', ', $result['unreadable']));
|
||||
$output->writeln('Last read error: ' . $result['readError']);
|
||||
}
|
||||
if ($result['missing'] === [] && $result['failed'] === [] && $result['unreadable'] === []) {
|
||||
$output->writeln('<info>OK</info>');
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -168,6 +168,7 @@ final class AdminApiController extends Controller {
|
||||
return new JSONResponse($this->targets->availableBackends());
|
||||
}
|
||||
|
||||
#[PasswordConfirmationRequired]
|
||||
public function addTarget(string $name, string $backend, string $auth, array $options = [], string $path = 'ng_backup', ?float $maxGb = null): JSONResponse {
|
||||
try {
|
||||
$r = $this->targets->add($name, $backend, $auth, $options, $path);
|
||||
|
||||
@@ -96,6 +96,12 @@ final class KeyRing {
|
||||
/** @param array{v:int, salt:string, ops:int, mem:int, nonce:string, key:string} $wrapped */
|
||||
public static function unwrap(array $wrapped, #[\SensitiveParameter] string $passphrase): self {
|
||||
$b64 = static fn (string $s): string => sodium_base642bin($s, SODIUM_BASE64_VARIANT_ORIGINAL);
|
||||
// ops/mem come from unauthenticated JSON (kit file, repository config): bound them, so a
|
||||
// damaged or crafted file cannot make the KDF use unbounded memory or time.
|
||||
if ($wrapped['ops'] < SODIUM_CRYPTO_PWHASH_OPSLIMIT_INTERACTIVE || $wrapped['ops'] > SODIUM_CRYPTO_PWHASH_OPSLIMIT_SENSITIVE
|
||||
|| $wrapped['mem'] < SODIUM_CRYPTO_PWHASH_MEMLIMIT_INTERACTIVE || $wrapped['mem'] > SODIUM_CRYPTO_PWHASH_MEMLIMIT_SENSITIVE) {
|
||||
throw new CryptoException('Unsupported key derivation parameters (damaged or foreign key file)');
|
||||
}
|
||||
$kek = sodium_crypto_pwhash(SODIUM_CRYPTO_SECRETBOX_KEYBYTES, $passphrase, $b64($wrapped['salt']),
|
||||
$wrapped['ops'], $wrapped['mem'], SODIUM_CRYPTO_PWHASH_ALG_ARGON2ID13);
|
||||
$master = sodium_crypto_secretbox_open($b64($wrapped['key']), $b64($wrapped['nonce']), $kek);
|
||||
|
||||
@@ -122,6 +122,7 @@ final class StreamCipher {
|
||||
/** fread() may return short reads on network streams; keep reading until $length or EOF. */
|
||||
private static function readFull($in, int $length): string {
|
||||
$buffer = '';
|
||||
$empty = 0;
|
||||
while (strlen($buffer) < $length && !feof($in)) {
|
||||
$chunk = fread($in, $length - strlen($buffer));
|
||||
if ($chunk === false) {
|
||||
@@ -132,8 +133,15 @@ final class StreamCipher {
|
||||
if (feof($in)) {
|
||||
break;
|
||||
}
|
||||
// A stalled network stream returns '' without EOF forever: back off, then give up
|
||||
// (about a minute) instead of spinning at full CPU until the lease expires.
|
||||
if (++$empty > 600) {
|
||||
throw new CryptoException('Read stalled');
|
||||
}
|
||||
usleep(100000);
|
||||
continue;
|
||||
}
|
||||
$empty = 0;
|
||||
$buffer .= $chunk;
|
||||
}
|
||||
return $buffer;
|
||||
|
||||
+52
-38
@@ -72,50 +72,57 @@ final class DbDumper {
|
||||
public function dumpStep(Repository $repo, array $state, float $deadline, array &$stats): array {
|
||||
$state['steps']++;
|
||||
$sm = $this->db->createSchemaManager();
|
||||
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
|
||||
$this->db->beginTransaction();
|
||||
// A pack that fills up during the dump is uploaded right away; its index cache row must
|
||||
// wait until the read-only transaction is over.
|
||||
$repo->holdIndexCache();
|
||||
try {
|
||||
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
|
||||
$this->db->executeStatement('SET TRANSACTION READ ONLY');
|
||||
}
|
||||
while ($state['i'] < count($state['tables'])) {
|
||||
$table = $state['tables'][$state['i']];
|
||||
$schema = $sm->introspectTable($table);
|
||||
$writer = $repo->blobWriter($stats);
|
||||
$hash = $state['hash'] !== null ? unserialize(base64_decode($state['hash']), ['allowed_classes' => [\HashContext::class]]) : hash_init('sha256');
|
||||
if ($state['after'] === null && $state['segments'] === []) {
|
||||
$writer->write($this->header($schema) . "\n");
|
||||
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
|
||||
$this->db->beginTransaction();
|
||||
try {
|
||||
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
|
||||
$this->db->executeStatement('SET TRANSACTION READ ONLY');
|
||||
}
|
||||
$complete = true;
|
||||
foreach ($this->rowLines($schema, $state['after']) as [$key, $line]) {
|
||||
$writer->write($line . "\n");
|
||||
hash_update($hash, $line . "\n");
|
||||
$state['rows']++;
|
||||
$state['after'] = $key;
|
||||
if (microtime(true) >= $deadline) {
|
||||
$complete = false;
|
||||
while ($state['i'] < count($state['tables'])) {
|
||||
$table = $state['tables'][$state['i']];
|
||||
$schema = $sm->introspectTable($table);
|
||||
$writer = $repo->blobWriter($stats);
|
||||
$hash = $state['hash'] !== null ? unserialize(base64_decode($state['hash']), ['allowed_classes' => [\HashContext::class]]) : hash_init('sha256');
|
||||
if ($state['after'] === null && $state['segments'] === []) {
|
||||
$writer->write($this->header($schema) . "\n");
|
||||
}
|
||||
$complete = true;
|
||||
foreach ($this->rowLines($schema, $state['after']) as [$key, $line]) {
|
||||
$writer->write($line . "\n");
|
||||
hash_update($hash, $line . "\n");
|
||||
$state['rows']++;
|
||||
$state['after'] = $key;
|
||||
if (microtime(true) >= $deadline) {
|
||||
$complete = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
$state['segments'][] = $writer->finish();
|
||||
$state['bytes'] += $writer->bytes();
|
||||
if (!$complete) {
|
||||
$state['hash'] = base64_encode(serialize($hash));
|
||||
break;
|
||||
}
|
||||
$state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'],
|
||||
'bytes' => $state['bytes'], 'sha256' => hash_final($hash)];
|
||||
$state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]);
|
||||
if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
$state['segments'][] = $writer->finish();
|
||||
$state['bytes'] += $writer->bytes();
|
||||
if (!$complete) {
|
||||
$state['hash'] = base64_encode(serialize($hash));
|
||||
break;
|
||||
if ($state['i'] >= count($state['tables'])) {
|
||||
$state['sequences'] = $this->sequenceStates($state['tables']);
|
||||
$state['done'] = true;
|
||||
}
|
||||
$state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'],
|
||||
'bytes' => $state['bytes'], 'sha256' => hash_final($hash)];
|
||||
$state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]);
|
||||
if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ($state['i'] >= count($state['tables'])) {
|
||||
$state['sequences'] = $this->sequenceStates($state['tables']);
|
||||
$state['done'] = true;
|
||||
} finally {
|
||||
$this->db->rollBack();
|
||||
}
|
||||
} finally {
|
||||
$this->db->rollBack();
|
||||
$repo->releaseIndexCache();
|
||||
}
|
||||
$repo->flushPacks();
|
||||
return $state;
|
||||
@@ -132,7 +139,7 @@ final class DbDumper {
|
||||
*
|
||||
* @param array{tables: array<string, array{blobs: list<string>, rows: int}>} $manifest
|
||||
*/
|
||||
public function restore(Repository $repo, array $manifest, ?array &$report = null): int {
|
||||
public function restore(Repository $repo, array $manifest, ?array &$report = null, ?callable $heartbeat = null): int {
|
||||
$sm = $this->db->createSchemaManager();
|
||||
$existing = array_flip($sm->listTableNames());
|
||||
$total = 0;
|
||||
@@ -175,6 +182,9 @@ final class DbDumper {
|
||||
if (count($batch) >= self::INSERT_ROWS) {
|
||||
$total += $this->insertBatch($table, $columns, $batch, $binary);
|
||||
$batch = [];
|
||||
if ($heartbeat !== null) {
|
||||
$heartbeat();
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($batch !== []) {
|
||||
@@ -253,7 +263,11 @@ final class DbDumper {
|
||||
} elseif (is_string($row[$i]) && !mb_check_encoding($row[$i], 'UTF-8')) {
|
||||
// Keep invalid UTF-8 byte-exact instead of substituting characters.
|
||||
$row[$i] = ['$b' => base64_encode($row[$i])];
|
||||
} elseif (is_int($row[$i]) || is_float($row[$i]) || is_bool($row[$i])) {
|
||||
} elseif (is_float($row[$i])) {
|
||||
// var_export uses serialize_precision (shortest exact form), not the lossy
|
||||
// 14-digit 'precision' of a (string) cast.
|
||||
$row[$i] = var_export($row[$i], true);
|
||||
} elseif (is_int($row[$i]) || is_bool($row[$i])) {
|
||||
// Normalise: drivers differ in returning numbers as int or string.
|
||||
$row[$i] = (string)(is_bool($row[$i]) ? (int)$row[$i] : $row[$i]);
|
||||
}
|
||||
|
||||
@@ -43,6 +43,17 @@ final class RunMapper extends QBMapper {
|
||||
return $this->findEntities($qb);
|
||||
}
|
||||
|
||||
/** @return list<Run> runs of $kind started at or after $since, newest first */
|
||||
public function startedSince(int $targetId, string $kind, int $since): array {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->select('*')->from($this->getTableName())
|
||||
->where($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)))
|
||||
->andWhere($qb->expr()->eq('kind', $qb->createNamedParameter($kind)))
|
||||
->andWhere($qb->expr()->gte('started_at', $qb->createNamedParameter($since, IQueryBuilder::PARAM_INT)))
|
||||
->orderBy('id', 'DESC');
|
||||
return $this->findEntities($qb);
|
||||
}
|
||||
|
||||
public function lastFinished(int $targetId, string $kind): ?Run {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->select('*')->from($this->getTableName())
|
||||
|
||||
+76
-16
@@ -49,9 +49,11 @@ final class BackupRun {
|
||||
public static function step(Repository $repo, array $state, float $deadline): array {
|
||||
$state['steps']++;
|
||||
if ($state['phase'] === 'files') {
|
||||
$walker = new TreeWalker($state['roots'], $state['exclude']);
|
||||
$parent = new ParentCursor($repo, $state['parent'], $state['parentPos']);
|
||||
$stats = $state['stats'];
|
||||
$walker = new TreeWalker($state['roots'], $state['exclude'], function (string $logical) use (&$state, &$stats): void {
|
||||
self::recordUnreadable($state, $stats, $logical);
|
||||
});
|
||||
$parent = new ParentCursor($repo, $state['parent'], $state['parentPos']);
|
||||
$lines = [];
|
||||
$finishedWalk = true;
|
||||
|
||||
@@ -68,15 +70,20 @@ final class BackupRun {
|
||||
$stats['reused']++;
|
||||
} else {
|
||||
$cur = $state['cur'];
|
||||
if ($cur === null || $cur['path'] !== $logical || $cur['version'] !== $version) {
|
||||
if ($cur !== null && $cur['path'] === $logical) {
|
||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1; // changed since the previous step
|
||||
if ($cur !== null && $cur['path'] === $logical && $cur['version'] !== $version) {
|
||||
// Changed since the previous step: start over, unless it keeps changing.
|
||||
if (self::countRetry($state, $stats, $logical)) {
|
||||
$cur['version'] = $version;
|
||||
} else {
|
||||
$cur = null;
|
||||
}
|
||||
}
|
||||
if ($cur === null || $cur['path'] !== $logical) {
|
||||
$cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version];
|
||||
}
|
||||
$fh = fopen($path, 'rb');
|
||||
$fh = @fopen($path, 'rb');
|
||||
if ($fh === false) {
|
||||
$stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1;
|
||||
self::recordUnreadable($state, $stats, $logical);
|
||||
$state['cur'] = null;
|
||||
continue;
|
||||
}
|
||||
@@ -91,12 +98,17 @@ final class BackupRun {
|
||||
}
|
||||
}
|
||||
fclose($fh);
|
||||
if (Repository::fileVersion($path) !== $cur['version']) {
|
||||
// Changed while reading: read it again from the start (now or next step).
|
||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
||||
$state['cur'] = null;
|
||||
$finishedWalk = false;
|
||||
break;
|
||||
$now = Repository::fileVersion($path);
|
||||
if ($now !== $cur['version']) {
|
||||
if (!self::countRetry($state, $stats, $logical)) {
|
||||
// Changed while reading: read it again from the start (now or next step).
|
||||
$state['cur'] = null;
|
||||
$finishedWalk = false;
|
||||
break;
|
||||
}
|
||||
// Keeps changing (a live database, VM image or log): keep what was read,
|
||||
// like any copy of a file in use, and report it instead of retrying forever.
|
||||
$cur['version'] = $now;
|
||||
}
|
||||
if (!$complete) {
|
||||
$state['cur'] = $cur;
|
||||
@@ -104,6 +116,7 @@ final class BackupRun {
|
||||
break;
|
||||
}
|
||||
$blobs = $cur['blobs'];
|
||||
$size = $cur['offset'];
|
||||
$state['cur'] = null;
|
||||
$stats['read']++;
|
||||
}
|
||||
@@ -133,19 +146,66 @@ final class BackupRun {
|
||||
|
||||
if ($state['phase'] === 'finish') {
|
||||
$segments = $state['segments'];
|
||||
$repo->writeSnapshot($state['snapshot'],
|
||||
$repo->putSnapshot($state['snapshot'],
|
||||
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
|
||||
'roots' => array_keys($state['roots']), 'stats' => $state['stats']] + $state['meta'],
|
||||
'roots' => array_keys($state['roots']), 'stats' => $state['stats'],
|
||||
'warnings' => self::warnings($state)] + $state['meta'],
|
||||
(function () use ($repo, $segments) {
|
||||
foreach ($segments as $blobs) {
|
||||
yield from $repo->readLines($blobs);
|
||||
}
|
||||
})());
|
||||
$state['phase'] = 'done';
|
||||
$state['phase'] = 'commit';
|
||||
}
|
||||
return $state;
|
||||
}
|
||||
|
||||
/**
|
||||
* A file or directory that could not be read: it is missing from this snapshot. Remembered (first
|
||||
* 1000, deduplicated across steps) so the run can report it instead of silently succeeding.
|
||||
*/
|
||||
private static function recordUnreadable(array &$state, array &$stats, string $logical): void {
|
||||
if (isset($state['unreadable'][$logical])) {
|
||||
return;
|
||||
}
|
||||
if (count($state['unreadable'] ?? []) < 1000) {
|
||||
$state['unreadable'][$logical] = true;
|
||||
}
|
||||
$stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1;
|
||||
}
|
||||
|
||||
/** @return array{unreadable: list<string>, changing: list<string>} examples for the report (first 50 each) */
|
||||
public static function warnings(array $state): array {
|
||||
return [
|
||||
'unreadable' => array_slice(array_map('strval', array_keys($state['unreadable'] ?? [])), 0, 50),
|
||||
'changing' => array_slice(array_map('strval', array_values($state['changing'] ?? [])), 0, 50),
|
||||
];
|
||||
}
|
||||
|
||||
/** Retries of one file that changes while it is read, before its last read is kept anyway. */
|
||||
public const MAX_RETRIES = 3;
|
||||
|
||||
/**
|
||||
* Count a retry of $logical. Returns true once it has changed MAX_RETRIES times: the caller then
|
||||
* keeps the data it read, and the file is listed in $state['changing'] (first 50) for the report.
|
||||
*/
|
||||
private static function countRetry(array &$state, array &$stats, string $logical): bool {
|
||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
||||
$n = (($state['retryPath'] ?? null) === $logical ? (int)($state['retryCount'] ?? 0) : 0) + 1;
|
||||
$state['retryPath'] = $logical;
|
||||
$state['retryCount'] = $n;
|
||||
if ($n < self::MAX_RETRIES) {
|
||||
return false;
|
||||
}
|
||||
if ($n === self::MAX_RETRIES) {
|
||||
$stats['changing'] = ($stats['changing'] ?? 0) + 1;
|
||||
if (count($state['changing'] ?? []) < 50) {
|
||||
$state['changing'][] = $logical;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static function readFull($fh, int $length): string {
|
||||
$buffer = '';
|
||||
while (strlen($buffer) < $length && !feof($fh)) {
|
||||
|
||||
@@ -23,6 +23,8 @@ final class TreeWalker {
|
||||
public function __construct(
|
||||
private array $roots,
|
||||
private array $exclude = [],
|
||||
/** Called with the logical path of a directory that cannot be listed. */
|
||||
private ?\Closure $onUnreadable = null,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -57,6 +59,9 @@ final class TreeWalker {
|
||||
}
|
||||
$names = @scandir($abs);
|
||||
if ($names === false) {
|
||||
if ($this->onUnreadable !== null) {
|
||||
($this->onUnreadable)($logical === '' ? '/' : $logical);
|
||||
}
|
||||
return;
|
||||
}
|
||||
$names = array_values(array_diff($names, ['.', '..']));
|
||||
|
||||
@@ -42,6 +42,10 @@ final class Notifier implements INotifier {
|
||||
: $l->t('Backup to %s failed', [$p['target']]));
|
||||
$notification->setParsedMessage((string)($p['error'] ?? ''));
|
||||
break;
|
||||
case 'run_warnings':
|
||||
$notification->setParsedSubject($l->t('Backup to %1$s finished with warnings: %2$s unreadable, %3$s changed while being read', [$p['target'], $p['unreadable'], $p['changing']]));
|
||||
$notification->setParsedMessage($l->t('Unreadable files and folders are missing from this backup (check their permissions). Files that kept changing were saved as they were while being read. Examples: %s', [$p['examples'] ?? '']));
|
||||
break;
|
||||
case 'quota_warning':
|
||||
$notification->setParsedSubject($l->t('Backup location %1$s is at %2$s%% of its limit (%3$s of %4$s)', [$p['target'], $p['percent'], $p['used'], $p['limit']]));
|
||||
$notification->setParsedMessage($l->t('Backups stop when the limit is reached. Raise the limit, or let the retention policy remove older backups.'));
|
||||
@@ -62,6 +66,8 @@ final class Notifier implements INotifier {
|
||||
'catalog_trusted' => $l->t('%1$s accepted an older state of backup location %2$s (catalog generation %3$s)', [$who, $p['target'] ?? '', $p['generation'] ?? '']),
|
||||
'slot_added' => $l->t('%1$s added backup passphrase slot %2$s (%3$s); download and confirm the new recovery kit', [$who, $p['slot'] ?? '', $p['label'] ?? '']),
|
||||
'slot_replaced' => $l->t('%1$s set a new passphrase for backup key slot %2$s (%3$s); download and confirm the new recovery kit', [$who, $p['slot'] ?? '', $p['label'] ?? '']),
|
||||
'user_export_retention_changed' => $l->t('%1$s changed how many exports per user are kept to %2$s', [$who, $p['keep'] ?? '']),
|
||||
'user_exports_forgotten' => $l->t('%1$s removed %2$s per-user exports from %3$s', [$who, $p['count'] ?? '', $p['target'] ?? '']),
|
||||
'slot_removed' => $l->t('%1$s removed backup passphrase slot %2$s; download and confirm the new recovery kit', [$who, $p['slot'] ?? '']),
|
||||
default => $l->t('NG Backup security event'),
|
||||
});
|
||||
|
||||
@@ -25,6 +25,8 @@ final class BlobIndex {
|
||||
private array $pending = [];
|
||||
|
||||
private ?IIndexCache $cache = null;
|
||||
/** @var list<\Closure():void>|null cache writes held back by holdCache(), or null when writing directly */
|
||||
private ?array $heldCacheWrites = null;
|
||||
private string $repositoryId = '';
|
||||
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
|
||||
public int $downloaded = 0;
|
||||
@@ -40,6 +42,46 @@ final class BlobIndex {
|
||||
$this->repositoryId = $repositoryId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hold back cache writes until releaseCache(). The database dump reads inside a read-only
|
||||
* transaction on the same connection that the cache writes to: a write there fails on
|
||||
* PostgreSQL and would be rolled back with the dump transaction elsewhere.
|
||||
*/
|
||||
public function holdCache(): void {
|
||||
$this->heldCacheWrites ??= [];
|
||||
}
|
||||
|
||||
/** Write the cache changes held back since holdCache(). */
|
||||
public function releaseCache(): void {
|
||||
$writes = $this->heldCacheWrites ?? [];
|
||||
$this->heldCacheWrites = null;
|
||||
foreach ($writes as $write) {
|
||||
$write();
|
||||
}
|
||||
}
|
||||
|
||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||
private function cachePut(string $packId, array $entries): void {
|
||||
if ($this->heldCacheWrites !== null) {
|
||||
$this->heldCacheWrites[] = function () use ($packId, $entries): void {
|
||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||
};
|
||||
return;
|
||||
}
|
||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||
}
|
||||
|
||||
/** @param list<string> $packIds */
|
||||
private function cacheRemove(array $packIds): void {
|
||||
if ($this->heldCacheWrites !== null) {
|
||||
$this->heldCacheWrites[] = function () use ($packIds): void {
|
||||
$this->cache?->remove($this->repositoryId, $packIds);
|
||||
};
|
||||
return;
|
||||
}
|
||||
$this->cache?->remove($this->repositoryId, $packIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the index: one listing of index/, entries from the local cache where available,
|
||||
* and only the index files that are not cached yet are downloaded. Cached packs that no
|
||||
@@ -50,7 +92,7 @@ final class BlobIndex {
|
||||
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
|
||||
$present = [];
|
||||
$this->downloaded = 0;
|
||||
foreach ($this->backend->list('index') as $path) {
|
||||
foreach ($this->backend->listFiles('index') as $path) {
|
||||
$packId = basename($path);
|
||||
$present[$packId] = true;
|
||||
if (isset($cached[$packId])) {
|
||||
@@ -112,7 +154,7 @@ final class BlobIndex {
|
||||
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
|
||||
rewind($stream);
|
||||
$this->backend->put('index/' . $packId, $stream);
|
||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||
$this->cachePut($packId, $entries);
|
||||
foreach ($entries as $e) {
|
||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||
unset($this->pending[$e['id']]);
|
||||
@@ -131,7 +173,7 @@ final class BlobIndex {
|
||||
/** Forget a pack: delete its index file and cache row. Blobs that now live elsewhere are kept. */
|
||||
public function removePack(string $packId): void {
|
||||
$this->backend->delete('index/' . $packId);
|
||||
$this->cache?->remove($this->repositoryId, [$packId]);
|
||||
$this->cacheRemove([$packId]);
|
||||
foreach ($this->blobs as $id => $loc) {
|
||||
if ($loc[0] === $packId) {
|
||||
unset($this->blobs[$id]);
|
||||
|
||||
@@ -28,6 +28,8 @@ final class Catalog {
|
||||
public array $snapshots = [];
|
||||
/** @var array<string, true> */
|
||||
public array $trash = [];
|
||||
/** @var array<string, true> manifest path => true, for a per-user export (users/<uid>/<exportId>) */
|
||||
public array $userExports = [];
|
||||
public string $hash = '';
|
||||
|
||||
public function __construct(
|
||||
@@ -39,7 +41,7 @@ final class Catalog {
|
||||
}
|
||||
|
||||
/** Load and verify the newest catalog; create the first one for repositories that have none. */
|
||||
public function load(callable $listSnapshots, callable $listTrash): void {
|
||||
public function load(callable $listSnapshots, callable $listTrash, callable $listUserExports): void {
|
||||
$gens = $this->generations();
|
||||
if ($gens === []) {
|
||||
// First use (new or pre-catalog repository): adopt the current state as generation 1.
|
||||
@@ -49,6 +51,7 @@ final class Catalog {
|
||||
}
|
||||
$this->snapshots = array_fill_keys($listSnapshots(), true);
|
||||
$this->trash = array_fill_keys($listTrash(), true);
|
||||
$this->userExports = array_fill_keys($listUserExports(), true);
|
||||
$this->write();
|
||||
return;
|
||||
}
|
||||
@@ -76,14 +79,23 @@ final class Catalog {
|
||||
$this->hash = $hash;
|
||||
$this->snapshots = array_fill_keys($data['snapshots'], true);
|
||||
$this->trash = array_fill_keys($data['trash'], true);
|
||||
// Catalogs written before user exports existed have no such key: default to empty rather
|
||||
// than adopting the location's current users/ listing (that would skip verification).
|
||||
$this->userExports = array_fill_keys($data['userExports'] ?? [], true);
|
||||
$this->anchor?->set($this->repositoryId, $this->gen, $this->hash);
|
||||
}
|
||||
|
||||
/** Write the next generation after the in-memory state was changed. */
|
||||
public function write(): void {
|
||||
$next = $this->gen + 1;
|
||||
// Two writers starting from the same generation would otherwise overwrite each other
|
||||
// silently and break the chain for whichever server anchored the lost write.
|
||||
if ($this->backend->exists($this->path($next))) {
|
||||
throw new RepositoryException("Catalog generation $next was written by another process at the same time; try again");
|
||||
}
|
||||
$plain = json_encode(['repository' => $this->repositoryId, 'gen' => $next, 'time' => gmdate('c'), 'prev' => $this->hash,
|
||||
'snapshots' => array_keys($this->snapshots), 'trash' => array_keys($this->trash)], JSON_THROW_ON_ERROR);
|
||||
'snapshots' => array_keys($this->snapshots), 'trash' => array_keys($this->trash),
|
||||
'userExports' => array_keys($this->userExports)], JSON_THROW_ON_ERROR);
|
||||
$encrypted = $this->cipher->encryptString($plain, $this->ad($next));
|
||||
$stream = fopen('php://memory', 'w+b');
|
||||
fwrite($stream, $encrypted);
|
||||
@@ -116,7 +128,7 @@ final class Catalog {
|
||||
|
||||
/** @return list<int> */
|
||||
private function generations(): array {
|
||||
return array_map(fn ($p) => (int)basename($p), $this->backend->list('catalog'));
|
||||
return array_map(fn ($p) => (int)basename($p), $this->backend->listFiles('catalog'));
|
||||
}
|
||||
|
||||
private function path(int $gen): string {
|
||||
|
||||
+279
-24
@@ -28,6 +28,8 @@ final class Repository {
|
||||
public const FORMAT = 1;
|
||||
public const BLOB_SIZE = 4 * 1048576;
|
||||
private const FLAG_DEFLATE = 1;
|
||||
/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
|
||||
private const VERIFY_READ_ATTEMPTS = 3;
|
||||
|
||||
private StreamCipher $cipher;
|
||||
private BlobIndex $index;
|
||||
@@ -90,8 +92,9 @@ final class Repository {
|
||||
private function loadCatalog(?ICatalogAnchor $anchor): void {
|
||||
$this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor);
|
||||
$this->catalog->load(
|
||||
fn () => array_map('basename', $this->backend->list('snapshots')),
|
||||
fn () => array_map('basename', $this->backend->list('trash/info')));
|
||||
fn () => array_map('basename', $this->backend->listFiles('snapshots')),
|
||||
fn () => array_map('basename', $this->backend->listFiles('trash/info')),
|
||||
fn () => $this->backend->list('users'));
|
||||
}
|
||||
|
||||
/** Refuse uploads that would make the stored data larger than this (null = no limit). */
|
||||
@@ -189,8 +192,15 @@ final class Repository {
|
||||
return ['snapshot' => $snapshotId] + $stats;
|
||||
}
|
||||
|
||||
/** Restore a snapshot (or only paths starting with $prefix) into a local directory. */
|
||||
public function restore(string $snapshotId, string $target, string $prefix = ''): int {
|
||||
/**
|
||||
* Restore a snapshot (or only paths starting with $prefix) into a local directory.
|
||||
*
|
||||
* @param bool $stripPrefix write "$target/<path without $prefix>" instead of
|
||||
* "$target/<full logical path>" (disaster recovery: restoring e.g. "data/" or
|
||||
* "config/" directly into the real data/config directory, not a subdirectory
|
||||
* named after the prefix). Ignored when $prefix is '' (nothing to strip).
|
||||
*/
|
||||
public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null): int {
|
||||
if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) {
|
||||
throw new RepositoryException('Invalid path prefix');
|
||||
}
|
||||
@@ -199,7 +209,8 @@ final class Repository {
|
||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||
continue;
|
||||
}
|
||||
$dest = rtrim($target, '/') . '/' . $entry['p'];
|
||||
$relPath = ($stripPrefix && $prefix !== '') ? substr($entry['p'], strlen(rtrim($prefix, '/')) + 1) : $entry['p'];
|
||||
$dest = rtrim($target, '/') . '/' . $relPath;
|
||||
if (!is_dir(dirname($dest))) {
|
||||
mkdir(dirname($dest), 0700, true);
|
||||
}
|
||||
@@ -218,14 +229,22 @@ final class Repository {
|
||||
}
|
||||
foreach ($entry['b'] as $id) {
|
||||
fwrite($out, $this->loadBlob($id));
|
||||
if ($heartbeat !== null) {
|
||||
$heartbeat();
|
||||
}
|
||||
}
|
||||
fclose($out);
|
||||
if (filesize($tmp) !== $entry['s']) {
|
||||
unlink($tmp);
|
||||
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
||||
}
|
||||
rename($tmp, $dest);
|
||||
touch($dest, $entry['m']);
|
||||
if (!rename($tmp, $dest)) {
|
||||
@unlink($tmp);
|
||||
throw new RepositoryException('Cannot put the restored file in place: ' . $entry['p']);
|
||||
}
|
||||
if (!touch($dest, $entry['m'])) {
|
||||
throw new RepositoryException('Cannot set the modification time of ' . $entry['p']);
|
||||
}
|
||||
$count++;
|
||||
}
|
||||
return $count;
|
||||
@@ -243,6 +262,15 @@ final class Repository {
|
||||
});
|
||||
}
|
||||
|
||||
/** See BlobIndex::holdCache(): no database writes until releaseIndexCache(). */
|
||||
public function holdIndexCache(): void {
|
||||
$this->index->holdCache();
|
||||
}
|
||||
|
||||
public function releaseIndexCache(): void {
|
||||
$this->index->releaseCache();
|
||||
}
|
||||
|
||||
public function flushPacks(): void {
|
||||
$this->runPacks?->flush();
|
||||
}
|
||||
@@ -279,6 +307,17 @@ final class Repository {
|
||||
* @param iterable<string> $lines
|
||||
*/
|
||||
public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
||||
$this->putSnapshot($snapshotId, $meta, $lines);
|
||||
$this->recordSnapshot($snapshotId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a snapshot object without touching the catalog. Use recordSnapshot() afterwards, from a
|
||||
* repository opened inside the catalog lock, so the catalog write builds on the latest generation.
|
||||
*
|
||||
* @param iterable<string> $lines
|
||||
*/
|
||||
public function putSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n");
|
||||
foreach ($lines as $line) {
|
||||
@@ -292,6 +331,9 @@ final class Repository {
|
||||
fclose($tree);
|
||||
rewind($encrypted);
|
||||
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
||||
}
|
||||
|
||||
public function recordSnapshot(string $snapshotId): void {
|
||||
$this->catalog->snapshots[$snapshotId] = true;
|
||||
$this->catalog->write();
|
||||
}
|
||||
@@ -349,31 +391,105 @@ final class Repository {
|
||||
// must be decrypted and re-encrypted under the new path, not moved as raw bytes.
|
||||
$dbInTrash = 'trash/' . $meta['db'];
|
||||
$this->putObject($dbInTrash, $this->getObject($meta['db']));
|
||||
$this->backend->delete($meta['db']);
|
||||
}
|
||||
// The snapshot object is bound to its id (not its path), so it can be moved as raw bytes.
|
||||
$this->backend->move('snapshots/' . $snapshotId, 'trash/snapshots/' . $snapshotId);
|
||||
// Copy first, switch the catalog, delete the originals last: a crash at any point leaves
|
||||
// either the old state or the new one plus unreferenced leftovers, never a catalog entry
|
||||
// pointing at a missing object. The snapshot object is bound to its id (not its path),
|
||||
// so it can be copied as raw bytes.
|
||||
$this->copyRaw('snapshots/' . $snapshotId, 'trash/snapshots/' . $snapshotId);
|
||||
$this->putObject('trash/info/' . $snapshotId, json_encode(['forgottenAt' => time(), 'by' => $by, 'db' => $meta['db'] ?? null,
|
||||
'dbInTrash' => $dbInTrash, 'time' => $meta['time'] ?? null, 'label' => $meta['label'] ?? ''], JSON_THROW_ON_ERROR));
|
||||
unset($this->catalog->snapshots[$snapshotId]);
|
||||
$this->catalog->trash[$snapshotId] = true;
|
||||
$this->catalog->write();
|
||||
if ($dbInTrash !== null) {
|
||||
$this->backend->delete($meta['db']);
|
||||
}
|
||||
$this->backend->delete('snapshots/' . $snapshotId);
|
||||
}
|
||||
|
||||
/** Undo forget(): move a snapshot back from the trash. */
|
||||
public function untrash(string $snapshotId): void {
|
||||
$info = $this->trashInfo($snapshotId);
|
||||
if ($info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash'])) {
|
||||
$restoreDb = $info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash']);
|
||||
if ($restoreDb) {
|
||||
$this->putObject($info['db'], $this->getObject($info['dbInTrash']));
|
||||
$this->backend->delete($info['dbInTrash']);
|
||||
}
|
||||
$this->backend->move('trash/snapshots/' . $snapshotId, 'snapshots/' . $snapshotId);
|
||||
// Same order as forget(): copy, switch the catalog, then delete the trash copies.
|
||||
$this->copyRaw('trash/snapshots/' . $snapshotId, 'snapshots/' . $snapshotId);
|
||||
$this->catalog->snapshots[$snapshotId] = true;
|
||||
unset($this->catalog->trash[$snapshotId]);
|
||||
$this->catalog->write();
|
||||
if ($restoreDb) {
|
||||
$this->backend->delete($info['dbInTrash']);
|
||||
}
|
||||
$this->backend->delete('trash/snapshots/' . $snapshotId);
|
||||
$this->backend->delete('trash/info/' . $snapshotId);
|
||||
}
|
||||
|
||||
/** Copy an object as raw bytes, buffered locally (remote read streams cannot be uploaded directly everywhere). */
|
||||
private function copyRaw(string $from, string $to): void {
|
||||
$in = $this->backend->get($from);
|
||||
$buffer = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
if ($buffer === false) {
|
||||
fclose($in);
|
||||
throw new RepositoryException("Cannot buffer $from");
|
||||
}
|
||||
try {
|
||||
if (stream_copy_to_stream($in, $buffer) === false) {
|
||||
throw new RepositoryException("Cannot read $from");
|
||||
}
|
||||
} finally {
|
||||
fclose($in);
|
||||
}
|
||||
rewind($buffer);
|
||||
$this->backend->put($to, $buffer);
|
||||
}
|
||||
|
||||
/** Record a user_migration export's manifest in the catalog (rollback-protected like a snapshot). */
|
||||
public function recordUserExport(string $manifestPath): void {
|
||||
$this->catalog->userExports[$manifestPath] = true;
|
||||
$this->catalog->write();
|
||||
}
|
||||
|
||||
/**
|
||||
* User export manifests according to the verified catalog, newest first. A manifest the
|
||||
* catalog lists but the location no longer has means it was deleted or hidden behind NG
|
||||
* Backup's back, same as a missing snapshot (see listSnapshots()).
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
public function listUserExports(?string $uid = null): array {
|
||||
$prefix = $uid !== null ? 'users/' . $uid . '/' : 'users/';
|
||||
$out = [];
|
||||
foreach (array_keys($this->catalog->userExports) as $path) {
|
||||
if (!str_starts_with($path, $prefix)) {
|
||||
continue;
|
||||
}
|
||||
if (!$this->backend->exists($path)) {
|
||||
throw new RepositoryException("User export $path is missing on the location (deleted or hidden outside NG Backup)");
|
||||
}
|
||||
$out[] = $path;
|
||||
}
|
||||
sort($out, SORT_STRING);
|
||||
return array_reverse($out);
|
||||
}
|
||||
|
||||
/** Whether $manifestPath is a user export the verified catalog actually recorded. */
|
||||
public function isRecordedUserExport(string $manifestPath): bool {
|
||||
return isset($this->catalog->userExports[$manifestPath]);
|
||||
}
|
||||
|
||||
/** Permanently delete a user export. Its blobs are freed by the next prune() like any other. */
|
||||
public function forgetUserExport(string $manifestPath): void {
|
||||
if (!isset($this->catalog->userExports[$manifestPath])) {
|
||||
throw new RepositoryException("$manifestPath is not a recorded user export");
|
||||
}
|
||||
unset($this->catalog->userExports[$manifestPath]);
|
||||
$this->catalog->write();
|
||||
$this->backend->delete($manifestPath);
|
||||
}
|
||||
|
||||
/** @return array<string, array{forgottenAt:int, by:string, db:?string, dbInTrash:?string, time:?string, label:string}> */
|
||||
public function trash(): array {
|
||||
$out = [];
|
||||
@@ -389,23 +505,26 @@ final class Repository {
|
||||
|
||||
/** Permanently delete trash entries older than $delaySeconds. @return list<string> purged snapshot ids */
|
||||
public function purgeTrash(int $delaySeconds): array {
|
||||
$purged = [];
|
||||
$doomed = [];
|
||||
foreach ($this->trash() as $id => $info) {
|
||||
if (time() - $info['forgottenAt'] < $delaySeconds) {
|
||||
continue;
|
||||
}
|
||||
$doomed[$id] = $info;
|
||||
unset($this->catalog->trash[$id]);
|
||||
}
|
||||
if ($doomed === []) {
|
||||
return [];
|
||||
}
|
||||
$this->catalog->write();
|
||||
foreach ($doomed as $id => $info) {
|
||||
if ($info['dbInTrash'] !== null) {
|
||||
$this->backend->delete($info['dbInTrash']);
|
||||
}
|
||||
$this->backend->delete('trash/snapshots/' . $id);
|
||||
$this->backend->delete('trash/info/' . $id);
|
||||
unset($this->catalog->trash[$id]);
|
||||
$purged[] = $id;
|
||||
}
|
||||
if ($purged !== []) {
|
||||
$this->catalog->write();
|
||||
}
|
||||
return $purged;
|
||||
return array_keys($doomed);
|
||||
}
|
||||
|
||||
private function trashInfo(string $snapshotId): array {
|
||||
@@ -457,7 +576,10 @@ final class Repository {
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach ($this->backend->list('users') as $path) {
|
||||
// listUserExports() throws when a recorded export's manifest is missing: then stop, since
|
||||
// freeing its blobs would turn a tampering or storage alarm into irreversible deletion.
|
||||
$exports = array_unique(array_merge($this->listUserExports(), $this->backend->list('users')));
|
||||
foreach ($exports as $path) {
|
||||
foreach (json_decode($this->getObject($path), true, 512, JSON_THROW_ON_ERROR)['entries'] as $e) {
|
||||
foreach ($e['b'] ?? [] as $b) {
|
||||
$used[$b] = true;
|
||||
@@ -521,7 +643,7 @@ final class Repository {
|
||||
* @return list<string>
|
||||
*/
|
||||
public function listSnapshots(): array {
|
||||
$present = array_flip(array_map('basename', $this->backend->list('snapshots')));
|
||||
$present = array_flip(array_map('basename', $this->backend->listFiles('snapshots')));
|
||||
$listed = array_keys($this->catalog->snapshots);
|
||||
foreach ($listed as $id) {
|
||||
if (!isset($present[$id])) {
|
||||
@@ -585,8 +707,13 @@ final class Repository {
|
||||
}
|
||||
|
||||
private function loadBlob(string $id): string {
|
||||
[$pack, $offset, $length, $raw, $flags] = $this->index->get($id);
|
||||
$encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||
[$pack, $offset, $length] = $this->index->get($id);
|
||||
return $this->decodeBlob($id, $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length));
|
||||
}
|
||||
|
||||
/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */
|
||||
private function decodeBlob(string $id, string $encrypted): string {
|
||||
[, , , $raw, $flags] = $this->index->get($id);
|
||||
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
||||
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
||||
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
||||
@@ -609,6 +736,134 @@ final class Repository {
|
||||
return $this->streamEntries($snapshotId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checksum audit of a snapshot: every blob it references (files and the database dump, when
|
||||
* the snapshot has one) must be in the index, freshly re-downloaded and re-authenticated from
|
||||
* the location rather than trusted from the local cache (same reasoning as prune()), and its
|
||||
* pack must still be present. With $deep, each blob is also downloaded and decrypted, which
|
||||
* verifies its AEAD authentication tag and its content hash against the blob id (loadBlob()
|
||||
* already does both on every real read -- this just does it for everything in the snapshot,
|
||||
* deliberately, rather than relying on it to come up during a later restore). A blob
|
||||
* referenced by more than one file is only checked once; a missing database manifest is a
|
||||
* failure, not something to silently skip over.
|
||||
*
|
||||
* @param ?callable $heartbeat called after every blob (shallow) or every blob read (deep);
|
||||
* callers use it to keep a time-boxed lease alive during a large audit
|
||||
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
||||
*/
|
||||
public function verify(string $snapshotId, bool $deep = false, ?callable $heartbeat = null): array {
|
||||
$heartbeat ??= static function (): void {
|
||||
};
|
||||
$this->index->load(true);
|
||||
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
||||
$seen = [];
|
||||
$packSeen = []; // pack id => exists on the backend; avoids one exists() call per blob
|
||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||
$stats['filesChecked']++;
|
||||
foreach ($entry['b'] as $id) {
|
||||
if (isset($seen[$id])) {
|
||||
continue;
|
||||
}
|
||||
$seen[$id] = true;
|
||||
$this->verifyBlob($id, $deep, $stats, $packSeen);
|
||||
$heartbeat();
|
||||
}
|
||||
}
|
||||
$meta = $this->snapshotMeta($snapshotId);
|
||||
if (isset($meta['db']) && is_string($meta['db'])) {
|
||||
if (!$this->backend->exists($meta['db'])) {
|
||||
$stats['missing'][] = $meta['db'];
|
||||
} else {
|
||||
$dbManifest = json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
||||
foreach ($dbManifest['tables'] as $info) {
|
||||
foreach ($info['blobs'] as $id) {
|
||||
if (isset($seen[$id])) {
|
||||
continue;
|
||||
}
|
||||
$seen[$id] = true;
|
||||
$this->verifyBlob($id, $deep, $stats, $packSeen);
|
||||
$heartbeat();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Same checks as verify(), for a user export: every blob its manifest references must be in
|
||||
* the freshly re-authenticated index with its pack present; $deep also decrypts each.
|
||||
*
|
||||
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
||||
*/
|
||||
public function verifyUserExport(string $manifestPath, bool $deep = false, ?callable $heartbeat = null): array {
|
||||
if (!$this->isRecordedUserExport($manifestPath)) {
|
||||
throw new RepositoryException("$manifestPath is not a recorded user export for this location");
|
||||
}
|
||||
$heartbeat ??= static function (): void {
|
||||
};
|
||||
$this->index->load(true);
|
||||
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
||||
$seen = [];
|
||||
$packSeen = [];
|
||||
$manifest = json_decode($this->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
||||
foreach ($manifest['entries'] as $entry) {
|
||||
if (($entry['t'] ?? '') !== 'f') {
|
||||
continue;
|
||||
}
|
||||
$stats['filesChecked']++;
|
||||
foreach ($entry['b'] ?? [] as $id) {
|
||||
if (isset($seen[$id])) {
|
||||
continue;
|
||||
}
|
||||
$seen[$id] = true;
|
||||
$this->verifyBlob($id, $deep, $stats, $packSeen);
|
||||
$heartbeat();
|
||||
}
|
||||
}
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/** @param array<string, bool> $packSeen */
|
||||
private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen): void {
|
||||
if (!$this->index->has($id)) {
|
||||
$stats['missing'][] = $id;
|
||||
return;
|
||||
}
|
||||
[$pack] = $this->index->get($id);
|
||||
$packExists = $packSeen[$pack] ??= $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
|
||||
if (!$packExists) {
|
||||
$stats['missing'][] = $id;
|
||||
return;
|
||||
}
|
||||
$stats['blobsChecked']++;
|
||||
if (!$deep) {
|
||||
return;
|
||||
}
|
||||
// Reading and checking are kept apart: a read that fails is usually a network or
|
||||
// storage error, so it is retried and reported as unreadable (a pack that stays
|
||||
// unreadable may still be truncated). Only a blob that was read but does not
|
||||
// decrypt or match its id counts as failed: that is proof of damage.
|
||||
[, $offset, $length] = $this->index->get($id);
|
||||
$encrypted = null;
|
||||
for ($attempt = 1; $attempt <= self::VERIFY_READ_ATTEMPTS && $encrypted === null; $attempt++) {
|
||||
try {
|
||||
$encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||
} catch (\Exception $e) { // storage adapters throw their own exceptions; \Error (a bug) is not retried
|
||||
$stats['readError'] = $e->getMessage();
|
||||
}
|
||||
}
|
||||
if ($encrypted === null) {
|
||||
$stats['unreadable'][] = $id;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
$stats['bytesChecked'] += strlen($this->decodeBlob($id, $encrypted));
|
||||
} catch (\Throwable) {
|
||||
$stats['failed'][] = $id;
|
||||
}
|
||||
}
|
||||
|
||||
/** The meta record (first line) of a snapshot. */
|
||||
public function snapshotMeta(string $snapshotId): array {
|
||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||
|
||||
@@ -54,6 +54,13 @@ final class AlertService {
|
||||
['kind' => $run->getKind(), 'target' => $targetName, 'error' => mb_strimwidth((string)$run->getError(), 0, 300, '…')]);
|
||||
}
|
||||
|
||||
/** A backup that finished but is missing unreadable files, or holds files that changed while read. */
|
||||
public function runWarnings(Run $run, string $targetName, int $unreadable, int $changing, array $examples): void {
|
||||
$this->notifyAdmins('run_warnings', 'run-' . $run->getId(), ['target' => $targetName,
|
||||
'unreadable' => (string)$unreadable, 'changing' => (string)$changing,
|
||||
'examples' => mb_strimwidth(implode(', ', array_merge($examples['unreadable'], $examples['changing'])), 0, 300, '…')]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remember how much is stored on a location (for the settings page) and warn administrators
|
||||
* once a day when it is at 80% of its limit or more.
|
||||
|
||||
@@ -14,6 +14,7 @@ use OCA\NgBackup\Db\Snapshot;
|
||||
use OCA\NgBackup\Db\SnapshotMapper;
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCA\NgBackup\Job\BackupRun;
|
||||
use OCA\NgBackup\Repository\RepositoryException;
|
||||
use OCP\App\IAppManager;
|
||||
use OCP\IConfig;
|
||||
use OCP\IDBConnection;
|
||||
@@ -147,9 +148,29 @@ final class BackupService {
|
||||
}
|
||||
} else {
|
||||
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
|
||||
if ($state['files']['phase'] === 'commit') {
|
||||
// Catalog write under the catalog lock, from a repository opened inside it: a per-user
|
||||
// export or restore may have written the catalog since this step opened $repo.
|
||||
try {
|
||||
$this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300,
|
||||
fn () => $this->targets->repository($target)->recordSnapshot((string)$state['files']['snapshot']));
|
||||
$state['files']['phase'] = 'done';
|
||||
} catch (LockedException|RepositoryException) {
|
||||
// Another catalog write is in progress (or just raced us to the next
|
||||
// generation): the next step retries this commit on the fresh catalog.
|
||||
}
|
||||
}
|
||||
if ($state['files']['phase'] === 'done') {
|
||||
$this->recordSnapshot($run, $state['files']);
|
||||
$this->alerts->storedBytes($target, $repo->storedBytes());
|
||||
$stats = $state['files']['stats'];
|
||||
if (($stats['unreadable'] ?? 0) > 0 || ($stats['changing'] ?? 0) > 0) {
|
||||
$warnings = BackupRun::warnings($state['files']);
|
||||
$this->logger->warning(sprintf('NG Backup run %d to %s: %d unreadable (missing from the snapshot): %s; %d changed while being read: %s',
|
||||
$run->getId(), $target->getName(), $stats['unreadable'] ?? 0, implode(', ', $warnings['unreadable']),
|
||||
$stats['changing'] ?? 0, implode(', ', $warnings['changing'])));
|
||||
$this->alerts->runWarnings($run, $target->getName(), (int)($stats['unreadable'] ?? 0), (int)($stats['changing'] ?? 0), $warnings);
|
||||
}
|
||||
$run->setStatus(Run::DONE);
|
||||
$run->setPhase('done');
|
||||
$run->setFinishedAt(time());
|
||||
|
||||
@@ -71,6 +71,86 @@ final class KeyService {
|
||||
$this->cached = $keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* Disaster recovery on a fresh installation: adopt the master key from a recovery kit
|
||||
* (occ backup:key:kit on the original installation) instead of generating a new one.
|
||||
* Afterwards occ backup:target:add opens the existing repository on the location with this
|
||||
* key, instead of creating a new, empty one.
|
||||
*
|
||||
* @param array<string, mixed> $kit as produced by recoveryKit()
|
||||
* @throws \OCA\NgBackup\Crypto\CryptoException wrong passphrase, or a damaged/foreign kit
|
||||
*/
|
||||
public function importRecoveryKit(array $kit, #[\SensitiveParameter] string $passphrase): void {
|
||||
if ($this->isInitialized()) {
|
||||
throw new \RuntimeException('A backup key already exists');
|
||||
}
|
||||
if (!isset($kit['wrapped_key'])) {
|
||||
throw new \InvalidArgumentException('Not a NG Backup recovery kit file');
|
||||
}
|
||||
$keys = KeyRing::unwrapAny($kit['wrapped_key'], $passphrase);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
||||
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
||||
$this->cached = $keys;
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, json_encode($kit['wrapped_key'], JSON_THROW_ON_ERROR), true, true);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
||||
// Kits from before this field existed fall back to the default.
|
||||
$delay = (int)($kit['delete_delay_days'] ?? self::DEFAULT_DELETE_DELAY);
|
||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, ($delay >= 1 && $delay <= 365) ? $delay : self::DEFAULT_DELETE_DELAY, true);
|
||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, (int)($kit['kit_version'] ?? 1), true);
|
||||
// Having and successfully using a real kit file is stronger proof than the normal
|
||||
// download-and-confirm flow it replaces.
|
||||
$this->confirmRecoveryKit('cli-import');
|
||||
}
|
||||
|
||||
/**
|
||||
* The master key and the recovery kit code are stored encrypted with Nextcloud's own
|
||||
* instance secret (ICrypto), not with themselves; the wrapped key is not (it is sealed with
|
||||
* the admin's passphrase instead), but Nextcloud's own AppConfig additionally encrypts any
|
||||
* "sensitive" value with the instance secret regardless, underneath whatever we store. So
|
||||
* changing that secret (occ backup:restore:full merging config.php) would otherwise leave
|
||||
* all three permanently undecryptable. Call $applySecretChange (which must be the only thing
|
||||
* that changes the secret) through here instead: all three are decrypted first, then
|
||||
* re-encrypted under whatever secret is active once $applySecretChange returns.
|
||||
*
|
||||
* Not crash-safe: a process kill between the delete and the re-save of one of these keys
|
||||
* would leave it missing rather than merely stale. Disaster recovery is run once, by hand,
|
||||
* right after occ maintenance:install; a dedicated two-phase write is not justified yet for
|
||||
* that narrow a window.
|
||||
*/
|
||||
public function reencryptForSecretRotation(callable $applySecretChange): mixed {
|
||||
$master = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true);
|
||||
$wrapped = $this->appConfig->getValueString(Application::APP_ID, self::KEY_WRAPPED, '', true);
|
||||
$kitCodeRaw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_KIT_CODE, '', true);
|
||||
$kitCode = $kitCodeRaw !== '' ? json_decode($kitCodeRaw, true, 512, JSON_THROW_ON_ERROR) : null;
|
||||
$masterPlain = $master !== '' ? $this->crypto->decrypt($master) : null;
|
||||
$kitCodePlain = $kitCode !== null && isset($kitCode['code']) ? $this->crypto->decrypt($kitCode['code']) : null;
|
||||
|
||||
$result = $applySecretChange();
|
||||
$this->cached = null; // the master key is about to be re-saved; do not keep the pre-rotation copy
|
||||
|
||||
// Nextcloud's own AppConfig additionally encrypts a "sensitive" value itself (tied to
|
||||
// the same secret, underneath our own encrypt() above); setValueString() on an existing
|
||||
// key reads the old value first to compare, which would try to decrypt it with the new
|
||||
// secret and fail the same way. Delete first so there is nothing to compare against.
|
||||
if ($masterPlain !== null) {
|
||||
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_MASTER);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER, $this->crypto->encrypt($masterPlain), true, true);
|
||||
}
|
||||
if ($wrapped !== '') {
|
||||
// Already plaintext JSON here: getValueString() above transparently undid
|
||||
// AppConfig's own sensitive-value encryption; there is no encrypt() of our own to
|
||||
// redo (wrapping is sealed with the passphrase, not the instance secret).
|
||||
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_WRAPPED);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, $wrapped, true, true);
|
||||
}
|
||||
if ($kitCodePlain !== null) {
|
||||
$kitCode['code'] = $this->crypto->encrypt($kitCodePlain);
|
||||
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_KIT_CODE);
|
||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_KIT_CODE, json_encode($kitCode, JSON_THROW_ON_ERROR), true, true);
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function keyRing(): KeyRing {
|
||||
if ($this->cached === null) {
|
||||
$enc = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true);
|
||||
@@ -182,13 +262,14 @@ final class KeyService {
|
||||
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_FINGERPRINT, '', true);
|
||||
}
|
||||
|
||||
/** @return array{version:int, app:string, fingerprint:string, created:string, wrapped_key:array, instructions:string} */
|
||||
/** @return array{version:int, app:string, fingerprint:string, kit_version:int, delete_delay_days:int, confirmation_code:string, created:string, wrapped_key:array, slots:list<array{slot:int, label:string, created:int}>, instructions:string} */
|
||||
public function recoveryKit(): array {
|
||||
return [
|
||||
'version' => 1,
|
||||
'app' => 'NG Backup',
|
||||
'fingerprint' => $this->fingerprint(),
|
||||
'kit_version' => $this->kitVersion(),
|
||||
'delete_delay_days' => $this->deleteDelayDays(),
|
||||
'confirmation_code' => $this->kitCode(),
|
||||
'created' => gmdate('c'),
|
||||
'wrapped_key' => $this->wrappedKey(),
|
||||
|
||||
@@ -76,6 +76,17 @@ final class LeaseService {
|
||||
$qb->executeStatement();
|
||||
}
|
||||
|
||||
/** Wraps $refresh so calling it often (per file, per row batch) only touches the database every $seconds. */
|
||||
public static function throttled(callable $refresh, int $seconds = 60): \Closure {
|
||||
$last = time();
|
||||
return static function () use ($refresh, $seconds, &$last): void {
|
||||
if (time() - $last >= $seconds) {
|
||||
$refresh();
|
||||
$last = time();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Run $fn under a lease; $refresh() can be called during long work. */
|
||||
public function with(string $name, string $mode, int $ttl, callable $fn): mixed {
|
||||
$id = $this->acquire($name, $mode, $ttl);
|
||||
|
||||
@@ -98,6 +98,83 @@ final class PruneService {
|
||||
return 'ng_backup/target/' . $target->getId();
|
||||
}
|
||||
|
||||
/**
|
||||
* A separate, short-lived exclusive lock serializing writes to the catalog itself. The
|
||||
* per-target lock above only keeps readers/writers out while a cleanup runs; it does not
|
||||
* stop two SHARED holders (e.g. two concurrent per-user backups, or one alongside a full
|
||||
* backup) from both reading the same catalog generation and racing to write the next one,
|
||||
* silently dropping whichever's change loses. The full-backup path avoids this today only
|
||||
* because a Run row makes concurrent runs for the same target mutually exclusive; per-user
|
||||
* backup/restore has no such row, so it takes this lock around the catalog write itself.
|
||||
*/
|
||||
public static function catalogLockKey(Target $target): string {
|
||||
return self::lockKey($target) . '/catalog';
|
||||
}
|
||||
|
||||
/** How many of each user's own exports (occ backup:user:backup) to keep, newest first. */
|
||||
public function userExportRetention(): int {
|
||||
return $this->appConfig->getValueInt(Application::APP_ID, 'user_export_retention', 3);
|
||||
}
|
||||
|
||||
public function setUserExportRetention(int $keep): void {
|
||||
if ($keep < 1) {
|
||||
throw new \InvalidArgumentException('Keep at least the last export');
|
||||
}
|
||||
$old = $this->userExportRetention();
|
||||
$this->appConfig->setValueInt(Application::APP_ID, 'user_export_retention', $keep);
|
||||
if ($old !== $keep) {
|
||||
$this->alerts->securityEvent('user_export_retention_changed', ['keep' => (string)$keep]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Forgets a user's older exports once more than userExportRetention() exist for them,
|
||||
* newest first; no trash, unlike a full snapshot (occ backup:user:backup is on-demand, not
|
||||
* yet the primary safety net the deletion delay protects).
|
||||
*
|
||||
* @return array{kept: array<string, list<string>>, forgotten: list<string>}
|
||||
*/
|
||||
public function applyUserExports(Target $target, bool $dryRun = false): array {
|
||||
if ($target->getAppendOnly()) {
|
||||
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
|
||||
}
|
||||
$lease = $this->lockExclusive($target, 1800);
|
||||
try {
|
||||
$repo = $this->targets->repository($target);
|
||||
$keep = $this->userExportRetention();
|
||||
$byUser = [];
|
||||
foreach ($repo->listUserExports() as $path) {
|
||||
$uid = explode('/', $path)[1] ?? '';
|
||||
$manifest = json_decode($repo->getObject($path), true, 512, JSON_THROW_ON_ERROR);
|
||||
$byUser[$uid][$path] = (int)strtotime((string)($manifest['time'] ?? '@0'));
|
||||
}
|
||||
$kept = [];
|
||||
$forget = [];
|
||||
foreach ($byUser as $uid => $times) {
|
||||
arsort($times);
|
||||
$i = 0;
|
||||
foreach (array_keys($times) as $path) {
|
||||
if ($i++ < $keep) {
|
||||
$kept[$uid][] = $path;
|
||||
} else {
|
||||
$forget[] = $path;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!$dryRun) {
|
||||
foreach ($forget as $path) {
|
||||
$repo->forgetUserExport($path);
|
||||
}
|
||||
if ($forget !== []) {
|
||||
$this->alerts->securityEvent('user_exports_forgotten', ['target' => $target->getName(), 'count' => (string)count($forget)], false);
|
||||
}
|
||||
}
|
||||
return ['kept' => $kept, 'forgotten' => $forget];
|
||||
} finally {
|
||||
$this->leases->release($lease);
|
||||
}
|
||||
}
|
||||
|
||||
public function policy(): RetentionPolicy {
|
||||
return RetentionPolicy::fromArray(json_decode($this->appConfig->getValueString(Application::APP_ID, 'retention', '{}'), true) ?: []);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\Db\DbDumper;
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCP\App\IAppManager;
|
||||
use OCP\IConfig;
|
||||
use OCP\IDBConnection;
|
||||
use OCP\Lock\LockedException;
|
||||
|
||||
/**
|
||||
* Disaster recovery on a fresh Nextcloud installation (same major version): replace the
|
||||
* database, restore the data directory and merge the backed-up config.php, see PLAN.md 3.7.
|
||||
*
|
||||
* Deliberately NOT handled yet (phase 3): restoring in place over a live, already-used
|
||||
* installation. The admin is expected to run this once, right after occ maintenance:install and
|
||||
* occ backup:key:import/backup:target:add, before anyone else touches the instance.
|
||||
*/
|
||||
final class RestoreFullService {
|
||||
/**
|
||||
* Config keys that must keep pointing at THIS installation's actual, already-working
|
||||
* database and data directory (set by occ maintenance:install), never at the old server's.
|
||||
*/
|
||||
private const CONNECTION_KEYS = [
|
||||
'dbtype', 'dbhost', 'dbport', 'dbname', 'dbuser', 'dbpassword', 'dbpersistent',
|
||||
'dbtableprefix', 'dbdriveroptions', 'datadirectory',
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private RestoreService $restore,
|
||||
private LeaseService $leases,
|
||||
private IConfig $config,
|
||||
private IDBConnection $db,
|
||||
private IAppManager $appManager,
|
||||
private KeyService $keys,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this installation can receive $snapshotId: same Nextcloud major version, and every
|
||||
* app that was enabled at backup time already installed here (its tables must exist before
|
||||
* the database restore can write into them).
|
||||
*
|
||||
* @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list<string>}
|
||||
*/
|
||||
public function checkCompatibility(Target $target, string $snapshotId): array {
|
||||
$meta = $this->targets->repository($target)->snapshotMeta($snapshotId);
|
||||
$currentMajor = explode('.', $this->config->getSystemValueString('version'))[0];
|
||||
$snapshotMajor = explode('.', (string)($meta['nextcloud'] ?? ''))[0];
|
||||
$enabled = array_flip($this->appManager->getEnabledApps());
|
||||
$missingApps = array_values(array_diff(array_keys($meta['apps'] ?? []), array_keys($enabled)));
|
||||
return [
|
||||
'ok' => $currentMajor === $snapshotMajor && $missingApps === [],
|
||||
'currentMajor' => $currentMajor,
|
||||
'snapshotMajor' => $snapshotMajor,
|
||||
'missingApps' => $missingApps,
|
||||
];
|
||||
}
|
||||
|
||||
/** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool} */
|
||||
public function restoreFull(Target $target, string $snapshotId): array {
|
||||
$check = $this->checkCompatibility($target, $snapshotId);
|
||||
if (!$check['ok']) {
|
||||
throw new \RuntimeException($this->describeBlockers($check));
|
||||
}
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId) {
|
||||
$beat = LeaseService::throttled($refresh);
|
||||
$repo = $this->targets->repository($target);
|
||||
$meta = $repo->snapshotMeta($snapshotId);
|
||||
|
||||
$dbReport = null;
|
||||
$manifest = [];
|
||||
if (isset($meta['db'])) {
|
||||
$manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
||||
$prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_');
|
||||
// ng_backup's own tables (and its own rows in the shared appconfig table) must
|
||||
// survive this restore untouched: they describe the backup/key/target that is
|
||||
// actively performing it right now, on THIS installation, not the one that was
|
||||
// backed up. Overwriting them mid-restore would pull the rug from under it.
|
||||
foreach (array_keys($manifest['tables']) as $table) {
|
||||
/** @var string $table */
|
||||
if (str_starts_with($table, $prefix . 'ngb_')) {
|
||||
unset($manifest['tables'][$table]);
|
||||
}
|
||||
}
|
||||
// Sequences are keyed separately from tables; excluding a table does not exclude
|
||||
// its sequence, which would otherwise rewind a retained ngb_* table's sequence
|
||||
// below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()).
|
||||
foreach (array_keys($manifest['sequences'] ?? []) as $seq) {
|
||||
/** @var string $seq */
|
||||
if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) {
|
||||
unset($manifest['sequences'][$seq]);
|
||||
}
|
||||
}
|
||||
$ownAppConfig = $this->preserveOwnAppConfig();
|
||||
$dumper = new DbDumper($this->db->getInner(), $prefix);
|
||||
try {
|
||||
$dumper->restore($repo, $manifest, $dbReport, $beat);
|
||||
} finally {
|
||||
$this->restoreOwnAppConfig($ownAppConfig);
|
||||
}
|
||||
}
|
||||
|
||||
// Read (but do not yet apply) the backed-up config.php before touching the data
|
||||
// directory: once applied, 'secret' may differ from what ng_backup's own Target
|
||||
// options were encrypted with (see restoreFull()'s docblock), and every further
|
||||
// repository access through $this->targets would fail to decrypt them.
|
||||
/** @var array<string, mixed> $backupConfig */
|
||||
$backupConfig = $this->fetchBackupConfig($target, $snapshotId);
|
||||
|
||||
$dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/');
|
||||
$dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat);
|
||||
|
||||
// Last step: nothing after this may need $this->targets->repository($target) again.
|
||||
if ($backupConfig !== []) {
|
||||
$this->targets->reencryptOptionsAround(
|
||||
fn (): mixed => $this->keys->reencryptForSecretRotation(fn (): mixed => $this->config->setSystemValues($backupConfig)),
|
||||
);
|
||||
}
|
||||
|
||||
return ['dbReport' => $dbReport ?? ['skipped' => [], 'restored' => []], 'dataFiles' => $dataFiles, 'configKeys' => count($backupConfig),
|
||||
'dbConsistent' => (bool)($manifest['consistent'] ?? true)];
|
||||
});
|
||||
}
|
||||
|
||||
/** @return array<string, mixed> config.php's $CONFIG from the backup, minus the connection keys */
|
||||
private function fetchBackupConfig(Target $target, string $snapshotId): array {
|
||||
$tmp = sys_get_temp_dir() . '/ngb-restore-config-' . bin2hex(random_bytes(6));
|
||||
try {
|
||||
$this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true);
|
||||
$path = $tmp . '/config.php';
|
||||
if (!is_file($path)) {
|
||||
return []; // nothing to merge; the live config.php stays as occ maintenance:install made it
|
||||
}
|
||||
$backupConfig = (static function () use ($path): array {
|
||||
$CONFIG = [];
|
||||
include $path;
|
||||
return $CONFIG;
|
||||
})();
|
||||
foreach (self::CONNECTION_KEYS as $key) {
|
||||
unset($backupConfig[$key]);
|
||||
}
|
||||
return $backupConfig;
|
||||
} finally {
|
||||
$this->rmrf($tmp);
|
||||
}
|
||||
}
|
||||
|
||||
/** @return list<array<string, mixed>> raw appconfig rows (appid, configkey, configvalue, type, lazy) */
|
||||
private function preserveOwnAppConfig(): array {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->select('*')->from('appconfig')
|
||||
->where($qb->expr()->eq('appid', $qb->createNamedParameter(\OCA\NgBackup\AppInfo\Application::APP_ID)));
|
||||
/** @var list<array<string, mixed>> */
|
||||
return $qb->executeQuery()->fetchAll();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param list<array<string, mixed>> $rows raw appconfig rows, as returned by preserveOwnAppConfig()
|
||||
*
|
||||
* Runs in its own transaction (DbDumper::restore() already committed its own by the time
|
||||
* this is called): not a full guarantee against a crash spanning both, but means this step
|
||||
* itself cannot leave ng_backup's appconfig rows half deleted, half reinserted.
|
||||
*/
|
||||
private function restoreOwnAppConfig(array $rows): void {
|
||||
$this->db->beginTransaction();
|
||||
try {
|
||||
$qb = $this->db->getQueryBuilder();
|
||||
$qb->delete('appconfig')
|
||||
->where($qb->expr()->eq('appid', $qb->createNamedParameter(\OCA\NgBackup\AppInfo\Application::APP_ID)));
|
||||
$qb->executeStatement();
|
||||
foreach ($rows as $row) {
|
||||
$ins = $this->db->getQueryBuilder();
|
||||
$ins->insert('appconfig')->values([
|
||||
'appid' => $ins->createNamedParameter($row['appid']),
|
||||
'configkey' => $ins->createNamedParameter($row['configkey']),
|
||||
'configvalue' => $ins->createNamedParameter($row['configvalue']),
|
||||
'type' => $ins->createNamedParameter($row['type'], \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_INT),
|
||||
'lazy' => $ins->createNamedParameter($row['lazy'], \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_INT),
|
||||
]);
|
||||
$ins->executeStatement();
|
||||
}
|
||||
$this->db->commit();
|
||||
} catch (\Throwable $e) {
|
||||
$this->db->rollBack();
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function rmrf(string $dir): void {
|
||||
if (!is_dir($dir)) {
|
||||
return;
|
||||
}
|
||||
foreach (new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir, \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST) as $f) {
|
||||
$f->isDir() ? rmdir($f->getPathname()) : unlink($f->getPathname());
|
||||
}
|
||||
rmdir($dir);
|
||||
}
|
||||
|
||||
private function describeBlockers(array $check): string {
|
||||
$parts = [];
|
||||
if ($check['currentMajor'] !== $check['snapshotMajor']) {
|
||||
$parts[] = "this installation is Nextcloud {$check['currentMajor']}, the backup is from {$check['snapshotMajor']} (same major version required)";
|
||||
}
|
||||
if ($check['missingApps'] !== []) {
|
||||
$parts[] = 'install these apps first (their tables must exist before the database restore can use them): '
|
||||
. implode(', ', array_map(fn (string $a) => "occ app:install $a", $check['missingApps']));
|
||||
}
|
||||
return implode('; ', $parts);
|
||||
}
|
||||
|
||||
private function withSharedLock(Target $target, callable $fn): mixed {
|
||||
try {
|
||||
// Refreshed during the database and data restore (per row batch, per blob), so a
|
||||
// cleanup cannot take over the location while a long restore is still reading it.
|
||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 7200, $fn);
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -149,15 +149,30 @@ final class RestoreService {
|
||||
return ['restored' => $restored, 'trashed' => $trashed, 'target' => $uid . ':' . $userFolder->getRelativePath($dest->getPath())];
|
||||
}
|
||||
|
||||
/** Raw restore of any snapshot path to a local directory (administrators, disaster recovery). */
|
||||
public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory): int {
|
||||
/**
|
||||
* Raw restore of any snapshot path to a local directory (administrators, disaster recovery).
|
||||
*
|
||||
* @param bool $stripPrefix write into $directory itself instead of $directory/$prefix (see
|
||||
* Repository::restore()); used to restore "data/" or "config/" straight into the
|
||||
* real data/config directory for occ backup:restore:full.
|
||||
*/
|
||||
public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null): int {
|
||||
if ($prefix !== '' && !Repository::isSafePath($prefix)) {
|
||||
throw new \InvalidArgumentException('Invalid path: ' . $prefix);
|
||||
}
|
||||
if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) {
|
||||
throw new \RuntimeException("Cannot create $directory");
|
||||
}
|
||||
return $this->withSharedLock($target, fn () => $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/'), 3600);
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat): int {
|
||||
$own = LeaseService::throttled($refresh);
|
||||
$beat = static function () use ($own, $heartbeat): void {
|
||||
$own();
|
||||
if ($heartbeat !== null) {
|
||||
$heartbeat();
|
||||
}
|
||||
};
|
||||
return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat);
|
||||
}, 3600);
|
||||
}
|
||||
|
||||
private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void {
|
||||
|
||||
@@ -10,10 +10,13 @@ namespace OCA\NgBackup\Service;
|
||||
use OCA\NgBackup\Backend\ExternalStorageFactory;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Backend\StorageBackend;
|
||||
use OCA\NgBackup\Db\Snapshot;
|
||||
use OCA\NgBackup\Db\SnapshotMapper;
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCA\NgBackup\Db\TargetMapper;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCP\App\IAppManager;
|
||||
use OCP\AppFramework\Db\DoesNotExistException;
|
||||
use OCP\Security\ICrypto;
|
||||
use OCP\Server;
|
||||
|
||||
@@ -30,6 +33,7 @@ final class TargetService {
|
||||
private IAppManager $appManager,
|
||||
private \OCA\NgBackup\Db\DbIndexCache $indexCache,
|
||||
private \OCA\NgBackup\Db\AppConfigCatalogAnchor $anchor,
|
||||
private SnapshotMapper $snapshots,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -49,6 +53,12 @@ final class TargetService {
|
||||
if (!preg_match('/^[A-Za-z0-9][A-Za-z0-9 _.-]{0,63}$/', $name)) {
|
||||
throw new \InvalidArgumentException('Name: letters, digits, space, _ . - (max 64)');
|
||||
}
|
||||
// Without the native extension files_external falls back to the smbclient binary, which
|
||||
// registers fine but cannot seek: the first backup then fails with "Seek failed". Refuse
|
||||
// up front instead of letting the admin find out from a failed run.
|
||||
if ($backend === 'smb' && !extension_loaded('smbclient')) {
|
||||
throw new \InvalidArgumentException('The SMB backend needs the PHP smbclient extension (pecl install smbclient); the smbclient binary alone cannot seek and every backup would fail');
|
||||
}
|
||||
try {
|
||||
$this->mapper->findByName($name);
|
||||
throw new \InvalidArgumentException("A target named '$name' already exists");
|
||||
@@ -78,7 +88,62 @@ final class TargetService {
|
||||
$target->setBasePath(trim($basePath, '/'));
|
||||
$target->setRepositoryId($repo->id());
|
||||
$target->setCreatedAt(time());
|
||||
return ['target' => $this->mapper->insert($target), 'created' => $created];
|
||||
$target = $this->mapper->insert($target);
|
||||
if (!$created) {
|
||||
// Disaster recovery: occ backup:list reads this local cache, not the repository
|
||||
// directly, so a snapshot made by another installation must be backfilled here too.
|
||||
$this->adoptSnapshots($target, $repo);
|
||||
}
|
||||
return ['target' => $target, 'created' => $created];
|
||||
}
|
||||
|
||||
/**
|
||||
* A target's options are encrypted with Nextcloud's own instance secret (ICrypto), not
|
||||
* ng_backup's key, so changing that secret (occ backup:restore:full merging config.php)
|
||||
* would otherwise leave every target permanently undecryptable. Call $applySecretChange
|
||||
* (which must be the only thing that changes the secret) through here instead: every
|
||||
* target's options are decrypted first, then re-encrypted under whatever secret is active
|
||||
* once $applySecretChange returns.
|
||||
*/
|
||||
public function reencryptOptionsAround(callable $applySecretChange): mixed {
|
||||
$plain = [];
|
||||
foreach ($this->mapper->findAll() as $target) {
|
||||
$plain[$target->getId()] = $this->options($target);
|
||||
}
|
||||
$result = $applySecretChange();
|
||||
foreach ($this->mapper->findAll() as $target) {
|
||||
if (!isset($plain[$target->getId()])) {
|
||||
continue; // added by $applySecretChange itself; nothing to re-encrypt
|
||||
}
|
||||
$target->setOptions($this->crypto->encrypt(json_encode($plain[$target->getId()], JSON_THROW_ON_ERROR)));
|
||||
$this->mapper->update($target);
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function adoptSnapshots(Target $target, Repository $repo): void {
|
||||
foreach ($repo->listSnapshots() as $snapshotId) {
|
||||
try {
|
||||
$this->snapshots->findOne($target->getId(), $snapshotId);
|
||||
continue; // already known locally
|
||||
} catch (DoesNotExistException) {
|
||||
}
|
||||
// One unreadable/corrupt snapshot must not stop the rest of the location's
|
||||
// snapshots from being backfilled, nor abort adding the target itself.
|
||||
try {
|
||||
$meta = $repo->snapshotMeta($snapshotId);
|
||||
$s = new Snapshot();
|
||||
$s->setTargetId($target->getId());
|
||||
$s->setSnapshotId($snapshotId);
|
||||
$s->setKind((string)($meta['kind'] ?? 'full'));
|
||||
$s->setLabel(($meta['label'] ?? '') !== '' ? $meta['label'] : null);
|
||||
$s->setCreatedAt((int)strtotime((string)($meta['time'] ?? 'now')));
|
||||
$s->setFiles((int)($meta['stats']['files'] ?? 0));
|
||||
$s->setBytes((int)($meta['stats']['bytes'] ?? 0));
|
||||
$this->snapshots->insert($s);
|
||||
} catch (\Throwable) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** @return list<Target> */
|
||||
@@ -87,7 +152,11 @@ final class TargetService {
|
||||
}
|
||||
|
||||
public function get(string $nameOrId): Target {
|
||||
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
||||
try {
|
||||
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
||||
} catch (DoesNotExistException) {
|
||||
throw new \InvalidArgumentException("No such location: $nameOrId");
|
||||
}
|
||||
}
|
||||
|
||||
/** Size limit for the data stored on this location (null = none). */
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCA\NgBackup\UserMigration\RepositoryExportDestination;
|
||||
use OCA\NgBackup\UserMigration\UserMigrationServiceLocator;
|
||||
use OCP\IUser;
|
||||
use OCP\Lock\LockedException;
|
||||
use OCP\UserMigration\UserMigrationException;
|
||||
|
||||
/**
|
||||
* Per-user backup via user_migration: every registered migrator's export streams straight into
|
||||
* the encrypted repository (no zip, no copy in the user's own storage, see
|
||||
* RepositoryExportDestination). Shares blobs with the full backup; prune() keeps a user export's
|
||||
* blobs referenced through its manifest under users/<uid>/.
|
||||
*
|
||||
* user_migration is an optional dependency (unlike files_external): without it, only this
|
||||
* per-user backup/restore is unavailable, the full backup still works.
|
||||
*/
|
||||
final class UserBackupService {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private LeaseService $leases,
|
||||
private UserMigrationServiceLocator $locator,
|
||||
) {
|
||||
}
|
||||
|
||||
/** @throws UserMigrationException when user_migration is not enabled, or the export itself fails */
|
||||
public function backupUser(Target $target, IUser $user): string {
|
||||
$service = $this->locator->get();
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $user, $service) {
|
||||
$repo = $this->targets->repository($target);
|
||||
$dest = new RepositoryExportDestination($repo, $user->getUID(), LeaseService::throttled($refresh));
|
||||
$service->export($dest, $user);
|
||||
// A short exclusive lock around the catalog write only: the shared lock above
|
||||
// allows another concurrent backup (full or per-user) to be writing the catalog at
|
||||
// the same time. The repository is reopened inside the lock, so the write builds on
|
||||
// the latest catalog generation instead of the one loaded before the lock was taken.
|
||||
$this->withCatalogLock($target, fn () => $this->targets->repository($target)->recordUserExport($dest->manifestPath()));
|
||||
return $dest->manifestPath();
|
||||
});
|
||||
}
|
||||
|
||||
private function withSharedLock(Target $target, callable $fn): mixed {
|
||||
try {
|
||||
// Refreshed by RepositoryExportDestination as it writes data, so a long export keeps it.
|
||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 3600, $fn);
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||
}
|
||||
}
|
||||
|
||||
private function withCatalogLock(Target $target, callable $fn): mixed {
|
||||
try {
|
||||
return $this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300, $fn);
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('Another catalog write for ' . $target->getName() . ' is in progress; try again');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCA\NgBackup\UserMigration\RepositoryExportDestination;
|
||||
use OCA\NgBackup\UserMigration\RepositoryImportSource;
|
||||
use OCA\NgBackup\UserMigration\UserMigrationServiceLocator;
|
||||
use OCP\EventDispatcher\IEventDispatcher;
|
||||
use OCP\Files\Events\Node\FilesystemTornDownEvent;
|
||||
use OCP\IUser;
|
||||
use OCP\IUserManager;
|
||||
use OCP\Lock\LockedException;
|
||||
use OCP\Security\ISecureRandom;
|
||||
use OCP\UserMigration\UserMigrationException;
|
||||
|
||||
/**
|
||||
* Restoring a user from a user_migration export (see UserBackupService), never over an existing
|
||||
* account:
|
||||
* - replace: a safety export of the current state goes into the repository first, then the
|
||||
* account is deleted and recreated fresh from the backup, under the same uid;
|
||||
* - as-backup: the backup is restored into a new account "<uid>-bak" (or "-bak-2", ...) instead,
|
||||
* so an admin can compare and transfer by hand. Nextcloud cannot rename a uid, so the *backup*
|
||||
* gets the new name, never the existing account.
|
||||
*/
|
||||
final class UserRestoreService {
|
||||
public const MODE_REPLACE = 'replace';
|
||||
public const MODE_AS_BACKUP = 'as-backup';
|
||||
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private LeaseService $leases,
|
||||
private UserMigrationServiceLocator $locator,
|
||||
private IUserManager $userManager,
|
||||
private ISecureRandom $random,
|
||||
private IEventDispatcher $events,
|
||||
) {
|
||||
}
|
||||
|
||||
/** @throws UserMigrationException when user_migration is not enabled, or the restore itself fails */
|
||||
public function restoreUser(Target $target, string $manifestPath, string $mode = self::MODE_REPLACE): IUser {
|
||||
$service = $this->locator->get();
|
||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $manifestPath, $mode, $service): IUser {
|
||||
$repo = $this->targets->repository($target);
|
||||
// Only ever read a manifest the verified catalog actually recorded: a path alone
|
||||
// (even one this app generated before) proves nothing about it still being current
|
||||
// or genuine once the catalog's rollback protection is supposed to vouch for it.
|
||||
if (!$repo->isRecordedUserExport($manifestPath)) {
|
||||
throw new \InvalidArgumentException("$manifestPath is not a recorded user export for this location");
|
||||
}
|
||||
$beat = LeaseService::throttled($refresh);
|
||||
$source = new RepositoryImportSource($repo, $manifestPath, $beat);
|
||||
$originalUid = $source->getOriginalUid();
|
||||
|
||||
if ($mode === self::MODE_AS_BACKUP) {
|
||||
$backupUid = $this->freeBackupUid($originalUid);
|
||||
$user = $this->userManager->createUser($backupUid, $this->random->generate(32));
|
||||
if ($user === false) {
|
||||
throw new \RuntimeException("Could not create the account $backupUid");
|
||||
}
|
||||
try {
|
||||
return $service->import($source, $user);
|
||||
} catch (\Throwable $e) {
|
||||
// A partially-imported, never-used account left behind is strictly worse
|
||||
// than no account at all; the original source account was never touched.
|
||||
$user->delete();
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
if ($mode !== self::MODE_REPLACE) {
|
||||
throw new \InvalidArgumentException("Unknown restore mode: $mode");
|
||||
}
|
||||
$existing = $this->userManager->get($originalUid);
|
||||
$safetyManifestPath = null;
|
||||
if ($existing !== null) {
|
||||
// Never import over a live account: export its current state first, so nothing
|
||||
// is lost if the restore turns out to be the wrong choice.
|
||||
$safety = new RepositoryExportDestination($repo, $originalUid, $beat);
|
||||
$service->export($safety, $existing);
|
||||
$safetyManifestPath = $safety->manifestPath();
|
||||
// Same race as UserBackupService::backupUser(): reopen the repository inside the
|
||||
// catalog lock so the write builds on the latest generation.
|
||||
$this->withCatalogLock($target, fn () => $this->targets->repository($target)->recordUserExport($safetyManifestPath));
|
||||
if (!$existing->delete()) {
|
||||
throw new \RuntimeException("Could not delete the existing account $originalUid before restoring; its safety export is at $safetyManifestPath");
|
||||
}
|
||||
// IRootFolder caches a Folder per uid for the lifetime of this process; without
|
||||
// this it would still hand the just-deleted account's (now storage-less) folder
|
||||
// to the import below, and every write would fail.
|
||||
$this->events->dispatchTyped(new FilesystemTornDownEvent());
|
||||
}
|
||||
try {
|
||||
return $service->import($source, null);
|
||||
} catch (\Throwable $e) {
|
||||
// The original account is already gone at this point (or never existed); a
|
||||
// failure here can only be reported, not undone in place. Its safety export
|
||||
// (if any) is the recovery path, already recorded above.
|
||||
throw new UserMigrationException(
|
||||
"Could not restore $originalUid" . ($safetyManifestPath !== null ? "; its safety export is at $safetyManifestPath" : '') . ': ' . $e->getMessage(),
|
||||
0,
|
||||
$e,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private function freeBackupUid(string $uid): string {
|
||||
$candidate = $uid . '-bak';
|
||||
for ($n = 2; $this->userManager->userExists($candidate); $n++) {
|
||||
$candidate = $uid . '-bak-' . $n;
|
||||
}
|
||||
return $candidate;
|
||||
}
|
||||
|
||||
private function withSharedLock(Target $target, callable $fn): mixed {
|
||||
try {
|
||||
// Refreshed while the safety export writes and the import reads (see UserBackupService::withSharedLock()).
|
||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 3600, $fn);
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||
}
|
||||
}
|
||||
|
||||
private function withCatalogLock(Target $target, callable $fn): mixed {
|
||||
try {
|
||||
return $this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300, $fn);
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('Another catalog write for ' . $target->getName() . ' is in progress; try again');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Service;
|
||||
|
||||
use OCA\NgBackup\Db\Target;
|
||||
use OCP\Lock\LockedException;
|
||||
|
||||
/** Checksum audit of a snapshot (occ backup:verify), see Repository::verify(). */
|
||||
final class VerifyService {
|
||||
public function __construct(
|
||||
private TargetService $targets,
|
||||
private LeaseService $leases,
|
||||
) {
|
||||
}
|
||||
|
||||
/** @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string} */
|
||||
public function verify(Target $target, string $snapshotId, bool $deep = false): array {
|
||||
return $this->run($target, $deep, fn (\OCA\NgBackup\Repository\Repository $repo, callable $heartbeat): array => $repo->verify($snapshotId, $deep, $heartbeat));
|
||||
}
|
||||
|
||||
/** @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string} */
|
||||
public function verifyUserExport(Target $target, string $manifestPath, bool $deep = false): array {
|
||||
return $this->run($target, $deep, fn (\OCA\NgBackup\Repository\Repository $repo, callable $heartbeat): array => $repo->verifyUserExport($manifestPath, $deep, $heartbeat));
|
||||
}
|
||||
|
||||
private function run(Target $target, bool $deep, callable $fn): array {
|
||||
$ttl = $deep ? 600 : 300;
|
||||
try {
|
||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, function (callable $refresh) use ($target, $deep, $ttl, $fn): array {
|
||||
$last = time();
|
||||
$heartbeat = function () use ($refresh, $ttl, &$last): void {
|
||||
if (time() - $last >= intdiv($ttl, 2)) {
|
||||
$refresh();
|
||||
$last = time();
|
||||
}
|
||||
};
|
||||
return $fn($this->targets->repository($target), $heartbeat);
|
||||
});
|
||||
} catch (LockedException) {
|
||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,8 @@ final class RepositoryExportDestination implements IExportDestination {
|
||||
public function __construct(
|
||||
private Repository $repo,
|
||||
private string $uid,
|
||||
/** Called while data is written (per chunk), to keep the location lease alive. */
|
||||
private ?\Closure $heartbeat = null,
|
||||
) {
|
||||
$this->exportId = gmdate('Ymd\THis\Z') . '-' . bin2hex(random_bytes(4));
|
||||
}
|
||||
@@ -36,6 +38,7 @@ final class RepositoryExportDestination implements IExportDestination {
|
||||
$writer = $this->repo->blobWriter($this->stats);
|
||||
$writer->write($content);
|
||||
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => strlen($content)];
|
||||
$this->beat();
|
||||
}
|
||||
|
||||
public function addFileAsStream(string $path, $stream): void {
|
||||
@@ -46,6 +49,7 @@ final class RepositoryExportDestination implements IExportDestination {
|
||||
throw new UserMigrationException('Read error while exporting ' . $path);
|
||||
}
|
||||
$writer->write($chunk);
|
||||
$this->beat();
|
||||
}
|
||||
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => $writer->bytes()];
|
||||
}
|
||||
@@ -65,8 +69,11 @@ final class RepositoryExportDestination implements IExportDestination {
|
||||
if ($stream === false) {
|
||||
throw new \RuntimeException('Cannot read ' . $node->getPath());
|
||||
}
|
||||
$this->addFileAsStream($target, $stream);
|
||||
fclose($stream);
|
||||
try {
|
||||
$this->addFileAsStream($target, $stream);
|
||||
} finally {
|
||||
fclose($stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -89,6 +96,12 @@ final class RepositoryExportDestination implements IExportDestination {
|
||||
return $this->stats;
|
||||
}
|
||||
|
||||
private function beat(): void {
|
||||
if ($this->heartbeat !== null) {
|
||||
($this->heartbeat)();
|
||||
}
|
||||
}
|
||||
|
||||
private static function norm(string $path): string {
|
||||
return trim($path, '/');
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ final class RepositoryImportSource implements IImportSource {
|
||||
public function __construct(
|
||||
private Repository $repo,
|
||||
string $manifestPath,
|
||||
/** Called while blobs are read, to keep the location lease alive. */
|
||||
private ?\Closure $heartbeat = null,
|
||||
) {
|
||||
$this->manifest = json_decode($repo->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
||||
}
|
||||
@@ -40,6 +42,9 @@ final class RepositoryImportSource implements IImportSource {
|
||||
throw new UserMigrationException('Cannot buffer ' . $path);
|
||||
}
|
||||
foreach ($this->repo->readBlobs($entry['b']) as $data) {
|
||||
if ($this->heartbeat !== null) {
|
||||
($this->heartbeat)();
|
||||
}
|
||||
for ($done = 0, $len = strlen($data); $done < $len; $done += $n) {
|
||||
$n = fwrite($stream, $done === 0 ? $data : substr($data, $done));
|
||||
if ($n === false || $n === 0) {
|
||||
@@ -109,7 +114,14 @@ final class RepositoryImportSource implements IImportSource {
|
||||
$destination->newFile($name, $stream);
|
||||
}
|
||||
} finally {
|
||||
fclose($stream);
|
||||
// putContent()/newFile() already close the stream once they've consumed it:
|
||||
// despite the @return resource docblock above, $stream is no longer a
|
||||
// resource by the time we get here (verified against a real storage; a
|
||||
// double fclose() throws a TypeError on PHP 8).
|
||||
/** @psalm-suppress RedundantConditionGivenDocblockType */
|
||||
if (is_resource($stream)) {
|
||||
fclose($stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\UserMigration;
|
||||
|
||||
use OCP\App\AppPathNotFoundException;
|
||||
use OCP\App\IAppManager;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Helper\QuestionHelper;
|
||||
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
||||
|
||||
/**
|
||||
* Installs or enables the optional user_migration app, but only when the administrator asked for it
|
||||
* (occ prompt or --install-user-migration). Runs occ in a child process, as the admin would.
|
||||
*/
|
||||
final class UserMigrationInstaller {
|
||||
private const APP = 'user_migration';
|
||||
|
||||
public function __construct(
|
||||
private IAppManager $appManager,
|
||||
) {
|
||||
}
|
||||
|
||||
public function isEnabled(): bool {
|
||||
return $this->appManager->isEnabledForAnyone(self::APP);
|
||||
}
|
||||
|
||||
private function codeIsPresent(): bool {
|
||||
try {
|
||||
$this->appManager->getAppPath(self::APP);
|
||||
return true;
|
||||
} catch (AppPathNotFoundException) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Installs (from the app store, if missing) and enables user_migration; throws on failure. */
|
||||
public function install(): void {
|
||||
$args = $this->codeIsPresent() ? ['app:enable', self::APP] : ['app:install', self::APP];
|
||||
/** @psalm-suppress UndefinedClass */
|
||||
$occ = \OC::$SERVERROOT . '/occ';
|
||||
$command = implode(' ', array_map('escapeshellarg', [PHP_BINARY, $occ, ...$args])) . ' 2>&1';
|
||||
exec($command, $lines, $code);
|
||||
if ($code !== 0) {
|
||||
throw new \RuntimeException("occ " . implode(' ', $args) . " failed:\n" . implode("\n", $lines));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns when user_migration is enabled. Otherwise asks the administrator (interactive shell) or
|
||||
* requires --install-user-migration, and throws when neither gives consent.
|
||||
*
|
||||
* @throws \RuntimeException
|
||||
*/
|
||||
public function ensureEnabled(InputInterface $input, OutputInterface $output): void {
|
||||
if ($this->isEnabled()) {
|
||||
return;
|
||||
}
|
||||
$consent = $input->getOption('install-user-migration') === true;
|
||||
if (!$consent && $input->isInteractive()) {
|
||||
$consent = (new QuestionHelper())->ask($input, $output, new ConfirmationQuestion(
|
||||
'The user_migration app is not enabled. Install and enable it now? [y/N] ', false)) === true;
|
||||
}
|
||||
if (!$consent) {
|
||||
throw new \RuntimeException('The user_migration app is not enabled; per-user backup/restore needs it. Run "occ '
|
||||
. 'app:install ' . self::APP . '" and "occ app:enable ' . self::APP . '", or repeat with --install-user-migration');
|
||||
}
|
||||
$output->writeln('Installing and enabling user_migration (approved by the administrator)...');
|
||||
$this->install();
|
||||
throw new \RuntimeException('user_migration is installed and enabled; run this command again');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\UserMigration;
|
||||
|
||||
use OCP\App\IAppManager;
|
||||
use OCP\Server;
|
||||
use OCP\UserMigration\UserMigrationException;
|
||||
|
||||
/**
|
||||
* Resolves user_migration's own (internal, non-public) UserMigrationService on demand.
|
||||
*
|
||||
* Unlike files_external, user_migration is an optional dependency: without it, only per-user
|
||||
* backup/restore is unavailable, the full backup still works. The class is never autoloaded by
|
||||
* composer; everything that touches it is behind this one class.
|
||||
*/
|
||||
final class UserMigrationServiceLocator {
|
||||
public function __construct(
|
||||
private IAppManager $appManager,
|
||||
) {
|
||||
}
|
||||
|
||||
/** @throws UserMigrationException when user_migration is not enabled anywhere */
|
||||
public function get(): object {
|
||||
if (!$this->appManager->isEnabledForAnyone('user_migration')) {
|
||||
throw new UserMigrationException('The user_migration app is not enabled; per-user backup/restore is unavailable');
|
||||
}
|
||||
$this->appManager->loadApp('user_migration');
|
||||
return Server::get(\OCA\UserMigration\Service\UserMigrationService::class);
|
||||
}
|
||||
}
|
||||
+16
-23
@@ -49,6 +49,11 @@
|
||||
<code><![CDATA[ask]]></code>
|
||||
</UndefinedInterfaceMethod>
|
||||
</file>
|
||||
<file src="lib/Command/KeyImport.php">
|
||||
<UndefinedInterfaceMethod>
|
||||
<code><![CDATA[ask]]></code>
|
||||
</UndefinedInterfaceMethod>
|
||||
</file>
|
||||
<file src="lib/Command/KeyInit.php">
|
||||
<UndefinedInterfaceMethod>
|
||||
<code><![CDATA[ask]]></code>
|
||||
@@ -219,14 +224,9 @@
|
||||
<MissingParamType>
|
||||
<code><![CDATA[$fh]]></code>
|
||||
</MissingParamType>
|
||||
<PossiblyNullIterator>
|
||||
<code><![CDATA[$segments]]></code>
|
||||
</PossiblyNullIterator>
|
||||
<PossiblyUndefinedArrayOffset>
|
||||
<code><![CDATA[$mtime]]></code>
|
||||
<code><![CDATA[$state['cur']]]></code>
|
||||
<code><![CDATA[$state['phase']]]></code>
|
||||
<code><![CDATA[$state['segments']]]></code>
|
||||
<code><![CDATA[$state['stats']]]></code>
|
||||
</PossiblyUndefinedArrayOffset>
|
||||
</file>
|
||||
@@ -409,24 +409,6 @@
|
||||
<ArgumentTypeCoercion>
|
||||
<code><![CDATA[array_diff(range(1, self::MAX_SLOTS), $used)]]></code>
|
||||
</ArgumentTypeCoercion>
|
||||
<InvalidReturnStatement>
|
||||
<code><![CDATA[[
|
||||
'version' => 1,
|
||||
'app' => 'NG Backup',
|
||||
'fingerprint' => $this->fingerprint(),
|
||||
'kit_version' => $this->kitVersion(),
|
||||
'confirmation_code' => $this->kitCode(),
|
||||
'created' => gmdate('c'),
|
||||
'wrapped_key' => $this->wrappedKey(),
|
||||
'slots' => $this->slots(),
|
||||
'instructions' => 'Keep this file safe, outside this server. Any one of the passphrases of the slots listed here opens it. '
|
||||
. 'To restore on a new server: install NG Backup, add the backup location and enter this kit with one passphrase. '
|
||||
. 'Without the kit (or the location) and a passphrase, the backups cannot be decrypted.',
|
||||
]]]></code>
|
||||
</InvalidReturnStatement>
|
||||
<InvalidReturnType>
|
||||
<code><![CDATA[array{version:int, app:string, fingerprint:string, created:string, wrapped_key:array, instructions:string}]]></code>
|
||||
</InvalidReturnType>
|
||||
<LessSpecificReturnStatement>
|
||||
<code><![CDATA[array_map(fn ($s) => ['slot' => $s['slot'], 'label' => $s['label'], 'created' => $s['created']], $this->wrappedKey()['slots'])]]></code>
|
||||
</LessSpecificReturnStatement>
|
||||
@@ -455,6 +437,17 @@
|
||||
<code><![CDATA[array{kept: array<string, list<string>>, forgotten: list<string>, purged: list<string>, prune: array}]]></code>
|
||||
</MoreSpecificReturnType>
|
||||
</file>
|
||||
<file src="lib/Service/RestoreFullService.php">
|
||||
<EmptyArrayAccess>
|
||||
<code><![CDATA[$backupConfig[$key]]]></code>
|
||||
</EmptyArrayAccess>
|
||||
<UndefinedInterfaceMethod>
|
||||
<code><![CDATA[getInner]]></code>
|
||||
</UndefinedInterfaceMethod>
|
||||
<UnresolvableInclude>
|
||||
<code><![CDATA[include $path]]></code>
|
||||
</UnresolvableInclude>
|
||||
</file>
|
||||
<file src="lib/Service/RestoreService.php">
|
||||
<LessSpecificReturnStatement>
|
||||
<code><![CDATA[array_values($children)]]></code>
|
||||
|
||||
@@ -31,8 +31,17 @@
|
||||
<errorLevel type="suppress">
|
||||
<!-- Provided by files_external at runtime; not a composer dependency (PLAN.md 3.5). -->
|
||||
<referencedClass name="OCA\Files_External\**" />
|
||||
<!-- Provided by user_migration at runtime, an optional dependency (PLAN.md 3.8). -->
|
||||
<referencedClass name="OCA\UserMigration\**" />
|
||||
</errorLevel>
|
||||
</UndefinedClass>
|
||||
<UndefinedDocblockClass>
|
||||
<errorLevel type="suppress">
|
||||
<!-- Server::get()'s templated return type resolving the users_migration class-string
|
||||
constant below; same reason as the UndefinedClass suppression above. -->
|
||||
<referencedClass name="OCA\UserMigration\**" />
|
||||
</errorLevel>
|
||||
</UndefinedDocblockClass>
|
||||
<!-- The #[\Override] attribute convention (PHP 8.3+) adds no behaviour; this app supports
|
||||
PHP 8.1+, and applying it to every interface-implementing method project-wide is pure
|
||||
ceremony with no bug-catching value, so it's suppressed rather than papered over. -->
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 221 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 238 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 226 KiB |
@@ -82,6 +82,7 @@ onBeforeUnmount(() => clearTimeout(timer))
|
||||
.ngb-tabs {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
margin: 0 0 12px;
|
||||
/* Same inline margin as NcSettingsSection, so the tabs line up with the section content */
|
||||
margin: 0 calc(var(--default-grid-baseline) * 7) 12px;
|
||||
}
|
||||
</style>
|
||||
|
||||
+27
-24
@@ -30,30 +30,32 @@
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<table v-if="status.runs.length" class="ngb-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>{{ t('ng_backup', 'Started') }}</th>
|
||||
<th>{{ t('ng_backup', 'Location') }}</th>
|
||||
<th>{{ t('ng_backup', 'Kind') }}</th>
|
||||
<th>{{ t('ng_backup', 'Restore point') }}</th>
|
||||
<th>{{ t('ng_backup', 'Status') }}</th>
|
||||
<th>{{ t('ng_backup', 'Files') }}</th>
|
||||
<th>{{ t('ng_backup', 'Uploaded') }}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="run in status.runs" :key="run.id" :title="run.error || ''">
|
||||
<td>{{ new Date(run.startedAt * 1000).toLocaleString() }}</td>
|
||||
<td>{{ targetName(run.targetId) }}</td>
|
||||
<td>{{ run.kind }}</td>
|
||||
<td>{{ run.snapshot ? new Date(run.snapshot.createdAt * 1000).toLocaleString() + (run.snapshot.label ? ' – ' + run.snapshot.label : '') : '' }}</td>
|
||||
<td :class="'ngb-' + run.status">{{ run.status }}{{ run.status === 'running' ? ' (' + run.phase + ')' : '' }}{{ run.error ? ': ' + run.error : '' }}</td>
|
||||
<td>{{ run.stats?.files?.files ?? run.stats?.restored ?? '' }}</td>
|
||||
<td>{{ run.stats?.files ? formatSize(run.stats.files.uploaded) : '' }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<div v-if="status.runs.length" class="ngb-table-wrap">
|
||||
<table class="ngb-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>{{ t('ng_backup', 'Started') }}</th>
|
||||
<th>{{ t('ng_backup', 'Location') }}</th>
|
||||
<th>{{ t('ng_backup', 'Kind') }}</th>
|
||||
<th>{{ t('ng_backup', 'Restore point') }}</th>
|
||||
<th>{{ t('ng_backup', 'Status') }}</th>
|
||||
<th>{{ t('ng_backup', 'Files') }}</th>
|
||||
<th>{{ t('ng_backup', 'Uploaded') }}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="run in status.runs" :key="run.id" :title="run.error || ''">
|
||||
<td>{{ new Date(run.startedAt * 1000).toLocaleString() }}</td>
|
||||
<td>{{ targetName(run.targetId) }}</td>
|
||||
<td>{{ run.kind }}</td>
|
||||
<td>{{ run.snapshot ? new Date(run.snapshot.createdAt * 1000).toLocaleString() + (run.snapshot.label ? ' – ' + run.snapshot.label : '') : '' }}</td>
|
||||
<td :class="'ngb-' + run.status">{{ run.status }}{{ run.status === 'running' ? ' (' + run.phase + ')' : '' }}{{ run.error ? ': ' + run.error : '' }}</td>
|
||||
<td>{{ run.stats?.files?.files ?? run.stats?.restored ?? '' }}</td>
|
||||
<td>{{ run.stats?.files ? formatSize(run.stats.files.uploaded) : '' }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</NcSettingsSection>
|
||||
</template>
|
||||
|
||||
@@ -126,6 +128,7 @@ async function saveSchedule() {
|
||||
<style scoped>
|
||||
.ngb-row { display: flex; gap: 8px; align-items: flex-end; flex-wrap: wrap; margin-bottom: 12px; }
|
||||
.ngb-time { max-width: 260px; }
|
||||
.ngb-table-wrap { overflow-x: auto; }
|
||||
.ngb-table { width: 100%; border-collapse: collapse; }
|
||||
.ngb-table th, .ngb-table td { text-align: start; padding: 4px 8px; border-bottom: 1px solid var(--color-border); }
|
||||
.ngb-failed { color: var(--color-error-text); }
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
<td>{{ t('ng_backup', 'Slot {n}', { n: slot.slot }) }}</td>
|
||||
<td><strong>{{ slot.label }}</strong></td>
|
||||
<td class="ngb-muted">{{ slot.created ? new Date(slot.created * 1000).toLocaleDateString() : '' }}</td>
|
||||
<td>
|
||||
<td class="ngb-actions">
|
||||
<NcButton variant="tertiary" @click="openSlotForm(slot)">{{ t('ng_backup', 'New passphrase') }}</NcButton>
|
||||
<NcButton v-if="status.key.slots.length > 1" variant="tertiary" @click="removeSlot(slot)">{{ t('ng_backup', 'Remove') }}</NcButton>
|
||||
</td>
|
||||
@@ -162,5 +162,6 @@ const confirm = () => guarded(async () => {
|
||||
.ngb-form, .ngb-confirm { display: flex; flex-direction: column; gap: 12px; max-width: 600px; margin: 8px 0; }
|
||||
.ngb-row { display: flex; gap: 8px; }
|
||||
.ngb-table td { padding: 2px 12px 2px 0; }
|
||||
.ngb-actions { display: flex; gap: 4px; }
|
||||
h3 { margin-top: 16px; font-weight: bold; }
|
||||
</style>
|
||||
|
||||
@@ -104,6 +104,7 @@ async function add() {
|
||||
busy.value = true
|
||||
error.value = ''
|
||||
try {
|
||||
await confirmPassword()
|
||||
await api.addTarget({ name: form.name, backend: backend.value.id, auth: auth.value.id, options: form.options, path: form.path,
|
||||
maxGb: form.maxGb === '' ? null : Number(form.maxGb) })
|
||||
adding.value = false
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
:description="t('ng_backup', 'Pick a restore point and a user folder or file. Restores run in the background and go through Nextcloud, so overwritten files keep their old content as a version and removed files go to the trash.')">
|
||||
<div class="ngb-row">
|
||||
<NcSelect v-model="target" :options="status.targets" label="name" :input-label="t('ng_backup', 'Location')" :clearable="false" />
|
||||
<NcSelect v-if="target" v-model="snapshot" :options="snapshots" label="displayLabel" :input-label="t('ng_backup', 'Restore point')" :clearable="false" />
|
||||
<NcSelect v-if="target" v-model="snapshot" :options="snapshots" label="displayLabel" class="ngb-snapshot" :input-label="t('ng_backup', 'Restore point')" :clearable="false" />
|
||||
</div>
|
||||
|
||||
<template v-if="snapshot">
|
||||
@@ -69,7 +69,7 @@ const busy = ref(false)
|
||||
const error = ref('')
|
||||
const message = ref('')
|
||||
const modes = [
|
||||
{ id: 'new-folder', label: t('ng_backup', 'Into a new folder "Restored <date>" (nothing existing is changed)') },
|
||||
{ id: 'new-folder', label: t('ng_backup', 'Into a new folder "Restored {date}" (nothing existing is changed)', { date: 'YYYY-MM-DD HH.MM' }) },
|
||||
{ id: 'merge', label: t('ng_backup', 'Into the original place, overwrite (old content kept as a version)') },
|
||||
{ id: 'replace', label: t('ng_backup', 'Into the original place, overwrite and move files that are not in the backup to the trash') },
|
||||
]
|
||||
@@ -124,6 +124,7 @@ async function restore() {
|
||||
|
||||
<style scoped>
|
||||
.ngb-row { display: flex; gap: 12px; flex-wrap: wrap; margin-bottom: 12px; }
|
||||
.ngb-row :deep(.ngb-snapshot) { min-width: min(360px, 100%); }
|
||||
.ngb-crumbs { display: flex; align-items: center; flex-wrap: wrap; }
|
||||
.ngb-entries li { display: flex; align-items: center; gap: 12px; min-height: 36px; }
|
||||
.ngb-entries li.selected { background: var(--color-primary-element-light); border-radius: var(--border-radius); }
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Backend;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\StorageBackend;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use OCP\Files\Storage\IStorage;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/** On a remote storage every is_dir() is a round trip: listing index/ must not cost one per pack. */
|
||||
class StorageBackendListTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
public function testListFilesChecksOnlyTheFolderItself(): void {
|
||||
$dir = $this->tempDir();
|
||||
foreach (['b1', 'a2', 'c3.part-0123abcd'] as $name) {
|
||||
touch("$dir/$name");
|
||||
}
|
||||
$isDirCalls = 0;
|
||||
$storage = $this->createMock(IStorage::class);
|
||||
$storage->method('instanceOfStorage')->willReturn(false);
|
||||
$storage->method('is_dir')->willReturnCallback(function (string $path) use (&$isDirCalls): bool {
|
||||
$isDirCalls++;
|
||||
return $path === 'repo' || $path === 'repo/index';
|
||||
});
|
||||
$storage->method('opendir')->willReturnCallback(fn (string $path) => $path === 'repo/index' ? opendir($dir) : false);
|
||||
|
||||
$backend = new StorageBackend($storage, 'repo');
|
||||
$isDirCalls = 0; // the constructor checks the base folder
|
||||
$this->assertSame(['index/a2', 'index/b1'], $backend->listFiles('index'));
|
||||
$this->assertSame(1, $isDirCalls);
|
||||
|
||||
$isDirCalls = 0;
|
||||
$this->assertSame(['index/a2', 'index/b1'], $backend->list('index'));
|
||||
$this->assertSame(4, $isDirCalls, 'list() checks every entry: the reason listFiles() exists');
|
||||
}
|
||||
|
||||
public function testListFilesOfAMissingFolderIsEmpty(): void {
|
||||
$storage = $this->createMock(IStorage::class);
|
||||
$storage->method('instanceOfStorage')->willReturn(false);
|
||||
$storage->method('is_dir')->willReturnCallback(fn (string $path): bool => $path === 'repo');
|
||||
$storage->expects($this->never())->method('opendir');
|
||||
$this->assertSame([], (new StorageBackend($storage, 'repo'))->listFiles('snapshots'));
|
||||
}
|
||||
|
||||
public function testAnUnreadableFolderIsAnErrorNotEmpty(): void {
|
||||
$storage = $this->createMock(IStorage::class);
|
||||
$storage->method('instanceOfStorage')->willReturn(false);
|
||||
$storage->method('is_dir')->willReturn(true);
|
||||
$storage->method('opendir')->willReturn(false);
|
||||
$this->expectException(BackendException::class);
|
||||
(new StorageBackend($storage, 'repo'))->listFiles('catalog');
|
||||
}
|
||||
}
|
||||
@@ -27,13 +27,14 @@ class BackupRunTest extends TestCase {
|
||||
|
||||
$state = BackupRun::start($src);
|
||||
$steps = 0;
|
||||
while ($state['phase'] !== 'done' && $steps < 500) {
|
||||
while ($state['phase'] !== 'commit' && $steps < 500) {
|
||||
// Fresh repository object per step, state through JSON: as separate processes would.
|
||||
$repo = Repository::openWithKey($backend, $keys);
|
||||
$state = json_decode(json_encode(BackupRun::step($repo, $state, microtime(true) + 0.05)), true);
|
||||
$steps++;
|
||||
}
|
||||
$this->assertSame('done', $state['phase']);
|
||||
$this->assertSame('commit', $state['phase']);
|
||||
Repository::openWithKey($backend, $keys)->recordSnapshot($state['snapshot']);
|
||||
$this->assertGreaterThan(2, $steps, 'the run was actually split into steps');
|
||||
|
||||
$target = $this->tempDir();
|
||||
@@ -50,7 +51,7 @@ class BackupRunTest extends TestCase {
|
||||
|
||||
$state = BackupRun::start($src);
|
||||
$mutated = false;
|
||||
for ($i = 0; $i < 500 && $state['phase'] !== 'done'; $i++) {
|
||||
for ($i = 0; $i < 500 && $state['phase'] !== 'commit'; $i++) {
|
||||
$state = BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true) + 0.01);
|
||||
if (!$mutated && $state['cur'] !== null && $state['cur']['offset'] > 0) {
|
||||
$fh = fopen("$src/big.bin", 'r+b');
|
||||
@@ -60,6 +61,8 @@ class BackupRunTest extends TestCase {
|
||||
$mutated = true;
|
||||
}
|
||||
}
|
||||
$this->assertSame('commit', $state['phase']);
|
||||
Repository::openWithKey($backend, $keys)->recordSnapshot($state['snapshot']);
|
||||
$this->assertTrue($mutated);
|
||||
$this->assertGreaterThanOrEqual(1, $state['stats']['retried'] ?? 0);
|
||||
$target = $this->tempDir();
|
||||
@@ -78,14 +81,17 @@ class BackupRunTest extends TestCase {
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
$run = function (?string $parent) use ($backend, $src, $keys): array {
|
||||
$state = BackupRun::start(['data' => $src], $parent);
|
||||
for ($i = 0; $i < 2000 && $state['phase'] !== 'done'; $i++) {
|
||||
for ($i = 0; $i < 2000 && $state['phase'] !== 'commit'; $i++) {
|
||||
$state = json_decode(json_encode(BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true))), true);
|
||||
}
|
||||
if ($state['phase'] === 'commit') {
|
||||
Repository::openWithKey($backend, $keys)->recordSnapshot($state['snapshot']);
|
||||
}
|
||||
return $state;
|
||||
};
|
||||
$first = $run(null);
|
||||
$second = $run($first['snapshot']);
|
||||
$this->assertSame('done', $second['phase']);
|
||||
$this->assertSame('commit', $second['phase']);
|
||||
$this->assertGreaterThan(150, $second["steps"], "one file per step");
|
||||
$this->assertSame(0, $second['stats']['read'], 'merge-join with the parent across steps reuses every file');
|
||||
$this->assertSame(200, $second['stats']['reused']);
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Job\BackupRun;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Repository\RepositoryException;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/** Data-safety cases found in the pre-beta review. */
|
||||
class CrashSafetyTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
public function testListingKeepsNamesThatOnlyLookLikeTemporaryFiles(): void {
|
||||
$root = $this->tempDir();
|
||||
$backend = new LocalBackend($root);
|
||||
mkdir("$root/users/john.part-time", 0700, true);
|
||||
file_put_contents("$root/users/john.part-time/20261006T120000Z-aaaa0001", 'x');
|
||||
file_put_contents("$root/users/upload.part-deadbeef", 'partial');
|
||||
$this->assertSame(['users/john.part-time/20261006T120000Z-aaaa0001'], $backend->list('users'));
|
||||
}
|
||||
|
||||
public function testPruneStopsWhenARecordedExportManifestIsMissing(): void {
|
||||
$root = $this->tempDir();
|
||||
$backend = new LocalBackend($root);
|
||||
$keys = KeyRing::generate();
|
||||
$repo = Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
$stats = [];
|
||||
$blob = $repo->storeData(random_bytes(1000), $stats);
|
||||
$repo->flushPacks();
|
||||
$path = 'users/alice/20261006T120000Z-aaaa0001';
|
||||
$repo->putObject($path, json_encode(['uid' => 'alice', 'entries' => ['f.txt' => ['t' => 'f', 'b' => [$blob], 's' => 1000]]]));
|
||||
$repo->recordUserExport($path);
|
||||
unlink("$root/$path");
|
||||
|
||||
$packsBefore = $backend->list('packs');
|
||||
try {
|
||||
Repository::openWithKey($backend, $keys)->prune();
|
||||
$this->fail('prune must refuse while a recorded export is missing');
|
||||
} catch (RepositoryException $e) {
|
||||
$this->assertStringContainsString('missing on the location', $e->getMessage());
|
||||
}
|
||||
$this->assertSame($packsBefore, $backend->list('packs'), 'no pack was deleted');
|
||||
}
|
||||
|
||||
public function testForgetAndUntrashKeepTheSnapshotRestorable(): void {
|
||||
$src = $this->tempDir();
|
||||
file_put_contents("$src/a.txt", 'hello');
|
||||
$root = $this->tempDir();
|
||||
$backend = new LocalBackend($root);
|
||||
$keys = KeyRing::generate();
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
$state = BackupRun::start($src);
|
||||
while ($state['phase'] !== 'commit') {
|
||||
$state = BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true) + 5);
|
||||
}
|
||||
Repository::openWithKey($backend, $keys)->recordSnapshot($state['snapshot']);
|
||||
$id = $state['snapshot'];
|
||||
|
||||
Repository::openWithKey($backend, $keys)->forget($id, 'test');
|
||||
$repo = Repository::openWithKey($backend, $keys);
|
||||
$this->assertSame([], $repo->listSnapshots());
|
||||
$this->assertArrayHasKey($id, $repo->trash());
|
||||
$this->assertFalse($backend->exists('snapshots/' . $id), 'original removed after the catalog switch');
|
||||
|
||||
$repo->untrash($id);
|
||||
$repo = Repository::openWithKey($backend, $keys);
|
||||
$this->assertSame([$id], $repo->listSnapshots());
|
||||
$this->assertSame([], $repo->trash());
|
||||
$target = $this->tempDir();
|
||||
$repo->restore($id, $target);
|
||||
$this->assertSame('hello', file_get_contents("$target/a.txt"));
|
||||
}
|
||||
|
||||
public function testFileThatKeepsChangingDoesNotStallTheRun(): void {
|
||||
$src = $this->tempDir();
|
||||
file_put_contents("$src/live.db", random_bytes(3 * Repository::BLOB_SIZE));
|
||||
$root = $this->tempDir();
|
||||
$backend = new LocalBackend($root);
|
||||
$keys = KeyRing::generate();
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
|
||||
$state = BackupRun::start($src);
|
||||
$n = 0;
|
||||
for ($i = 0; $i < 50 && $state['phase'] !== 'commit'; $i++) {
|
||||
$state = BackupRun::step(Repository::openWithKey($backend, $keys), $state, microtime(true) + 0.001);
|
||||
// Rewrite the file after every step: it never stays the same between two steps.
|
||||
$fh = fopen("$src/live.db", 'r+b');
|
||||
fwrite($fh, random_bytes(16));
|
||||
fclose($fh);
|
||||
touch("$src/live.db", time() + ++$n);
|
||||
clearstatcache();
|
||||
}
|
||||
$this->assertSame('commit', $state['phase'], 'the run finishes instead of retrying forever');
|
||||
$this->assertSame(1, $state['stats']['changing'] ?? 0);
|
||||
$this->assertSame(['live.db'], BackupRun::warnings($state)['changing']);
|
||||
}
|
||||
}
|
||||
@@ -51,4 +51,41 @@ class IndexCacheTest extends TestCase {
|
||||
$fourth->restore($r1['snapshot'], $target);
|
||||
$this->assertCount(12, self::hashTree($target));
|
||||
}
|
||||
|
||||
/** The database dump reads in a read-only transaction: a pack that fills up then must not write the cache yet. */
|
||||
public function testHeldCacheWritesWaitForRelease(): void {
|
||||
$cache = new class implements IIndexCache {
|
||||
public array $data = [];
|
||||
public bool $readOnly = false;
|
||||
public function all(string $r): array { return $this->data[$r] ?? []; }
|
||||
public function put(string $r, string $p, array $e): void {
|
||||
if ($this->readOnly) {
|
||||
throw new \RuntimeException('cannot execute INSERT in a read-only transaction');
|
||||
}
|
||||
$this->data[$r][$p] = $e;
|
||||
}
|
||||
public function remove(string $r, array $ps): void { foreach ($ps as $p) { unset($this->data[$r][$p]); } }
|
||||
};
|
||||
$backend = new LocalBackend($this->tempDir());
|
||||
$keys = KeyRing::generate();
|
||||
Repository::initWithKey($backend, $keys, $keys->wrap('pw'));
|
||||
$repo = Repository::openWithKey($backend, $keys, $cache);
|
||||
|
||||
$stats = [];
|
||||
$repo->holdIndexCache();
|
||||
$cache->readOnly = true;
|
||||
$writer = $repo->blobWriter($stats);
|
||||
for ($i = 0; $i < 40; $i++) {
|
||||
$writer->write(random_bytes(1048576)); // more than one 32 MiB pack
|
||||
}
|
||||
$writer->finish();
|
||||
$this->assertGreaterThan(0, count($backend->list('index')), 'a full pack was uploaded during the hold');
|
||||
$this->assertSame([], $cache->data);
|
||||
|
||||
$cache->readOnly = false;
|
||||
$repo->releaseIndexCache();
|
||||
$repo->flushPacks();
|
||||
$this->assertCount(count($backend->list('index')), array_merge(...array_values($cache->data)));
|
||||
$this->assertSame(0, Repository::openWithKey($backend, $keys, $cache)->indexDownloads());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Repository\ICatalogAnchor;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Repository\RepositoryException;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
/**
|
||||
* Repository::recordUserExport/listUserExports/forgetUserExport: the catalog side of a
|
||||
* user_migration export (see RepositoryExportDestination/RepositoryImportSource, which this
|
||||
* does not exercise directly since they implement OCP\UserMigration interfaces not available to
|
||||
* plain PHPUnit; that round trip is covered by tests/integration/user-roundtrip.sh instead).
|
||||
*/
|
||||
class UserExportTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private LocalBackend $backend;
|
||||
private KeyRing $keys;
|
||||
private ICatalogAnchor $anchor;
|
||||
private string $root;
|
||||
|
||||
protected function setUp(): void {
|
||||
$this->root = $this->tempDir();
|
||||
$this->backend = new LocalBackend($this->root);
|
||||
$this->keys = KeyRing::generate();
|
||||
$this->anchor = new class implements ICatalogAnchor {
|
||||
public array $data = [];
|
||||
public function get(string $r): ?array { return $this->data[$r] ?? null; }
|
||||
public function set(string $r, int $g, string $h): void { $this->data[$r] = ['gen' => $g, 'hash' => $h]; }
|
||||
};
|
||||
}
|
||||
|
||||
private function open(): Repository {
|
||||
return Repository::openWithKey($this->backend, $this->keys, null, $this->anchor);
|
||||
}
|
||||
|
||||
/** Stands in for an export destination's close(): a manifest object plus a catalog record. */
|
||||
private function putManifest(Repository $repo, string $path): void {
|
||||
$repo->putObject($path, json_encode(['uid' => 'alice']));
|
||||
$repo->recordUserExport($path);
|
||||
}
|
||||
|
||||
public function testRecordAndListSurviveAReopen(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
$this->putManifest($repo, 'users/alice/20261005T120000Z-aaaa0001');
|
||||
|
||||
$again = $this->open();
|
||||
$this->assertSame(['users/alice/20261005T120000Z-aaaa0001'], $again->listUserExports());
|
||||
$this->assertSame(['users/alice/20261005T120000Z-aaaa0001'], $again->listUserExports('alice'));
|
||||
$this->assertSame([], $again->listUserExports('bob'));
|
||||
}
|
||||
|
||||
public function testListIsNewestFirst(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
$this->putManifest($repo, 'users/alice/20261005T120000Z-aaaa0001');
|
||||
$this->putManifest($repo, 'users/alice/20261006T120000Z-aaaa0002');
|
||||
|
||||
$this->assertSame(
|
||||
['users/alice/20261006T120000Z-aaaa0002', 'users/alice/20261005T120000Z-aaaa0001'],
|
||||
$repo->listUserExports('alice'),
|
||||
);
|
||||
}
|
||||
|
||||
public function testForgetRemovesItFromTheCatalogAndLocation(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
$path = 'users/alice/20261005T120000Z-aaaa0001';
|
||||
$this->putManifest($repo, $path);
|
||||
|
||||
$repo->forgetUserExport($path);
|
||||
|
||||
$this->assertSame([], $repo->listUserExports());
|
||||
$this->assertFalse($this->backend->exists($path));
|
||||
}
|
||||
|
||||
public function testForgetRejectsAPathNotInTheCatalog(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
$this->expectException(RepositoryException::class);
|
||||
$repo->forgetUserExport('users/nobody/not-a-real-export');
|
||||
}
|
||||
|
||||
public function testIsRecordedUserExport(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
$path = 'users/alice/20261005T120000Z-aaaa0001';
|
||||
$this->putManifest($repo, $path);
|
||||
|
||||
$this->assertTrue($repo->isRecordedUserExport($path));
|
||||
$this->assertFalse($repo->isRecordedUserExport('users/alice/not-the-real-one'));
|
||||
}
|
||||
|
||||
public function testListingDetectsAManifestRemovedBehindItsBack(): void {
|
||||
// A catalog-listed export whose object disappeared from the location (deleted or
|
||||
// hidden outside NG Backup) must be detected, the same as a missing snapshot.
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'), $this->anchor);
|
||||
$path = 'users/alice/20261005T120000Z-aaaa0001';
|
||||
$this->putManifest($repo, $path);
|
||||
$this->backend->delete($path);
|
||||
|
||||
$this->expectException(RepositoryException::class);
|
||||
$this->expectExceptionMessage('missing on the location');
|
||||
$repo->listUserExports();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Repository;
|
||||
|
||||
use OCA\NgBackup\Backend\BackendException;
|
||||
use OCA\NgBackup\Backend\IBackend;
|
||||
use OCA\NgBackup\Backend\LocalBackend;
|
||||
use OCA\NgBackup\Crypto\KeyRing;
|
||||
use OCA\NgBackup\Repository\Repository;
|
||||
use OCA\NgBackup\Tests\Unit\TempDirTrait;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
class VerifyTest extends TestCase {
|
||||
use TempDirTrait;
|
||||
|
||||
private LocalBackend $backend;
|
||||
private KeyRing $keys;
|
||||
private string $root;
|
||||
private string $src;
|
||||
|
||||
protected function setUp(): void {
|
||||
$this->root = $this->tempDir();
|
||||
$this->backend = new LocalBackend($this->root);
|
||||
$this->keys = KeyRing::generate();
|
||||
$this->src = $this->tempDir();
|
||||
file_put_contents("$this->src/a", random_bytes(100000));
|
||||
file_put_contents("$this->src/b", random_bytes(100000));
|
||||
}
|
||||
|
||||
private function open(): Repository {
|
||||
return Repository::openWithKey($this->backend, $this->keys, null, null);
|
||||
}
|
||||
|
||||
public function testShallowVerifyPassesOnAHealthyRepository(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$s = $repo->backupDirectory($this->src);
|
||||
|
||||
$result = $this->open()->verify($s['snapshot']);
|
||||
|
||||
$this->assertSame(2, $result['filesChecked']);
|
||||
$this->assertGreaterThanOrEqual(1, $result['blobsChecked']);
|
||||
$this->assertSame(0, $result['bytesChecked']); // not downloaded without --deep
|
||||
$this->assertSame([], $result['missing']);
|
||||
$this->assertSame([], $result['failed']);
|
||||
}
|
||||
|
||||
public function testDeepVerifyDownloadsAndCountsBytes(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$s = $repo->backupDirectory($this->src);
|
||||
|
||||
$result = $this->open()->verify($s['snapshot'], deep: true);
|
||||
|
||||
$this->assertSame(200000, $result['bytesChecked']);
|
||||
$this->assertSame([], $result['missing']);
|
||||
$this->assertSame([], $result['failed']);
|
||||
}
|
||||
|
||||
public function testShallowVerifyDetectsAMissingPack(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$s = $repo->backupDirectory($this->src);
|
||||
foreach ($this->backend->list('packs') as $pack) {
|
||||
$this->backend->delete($pack);
|
||||
}
|
||||
|
||||
$result = $this->open()->verify($s['snapshot']);
|
||||
|
||||
$this->assertNotSame([], $result['missing']);
|
||||
}
|
||||
|
||||
public function testVerifyFailsOnAMissingDatabaseManifest(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$repo->writeSnapshot('snap-no-db', ['time' => gmdate('c'), 'db' => 'db/does-not-exist'], []);
|
||||
|
||||
$result = $this->open()->verify('snap-no-db');
|
||||
|
||||
$this->assertSame(['db/does-not-exist'], $result['missing']);
|
||||
}
|
||||
|
||||
public function testDeepVerifyDetectsATamperedPack(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$s = $repo->backupDirectory($this->src);
|
||||
$packs = $this->backend->list('packs');
|
||||
$this->assertNotEmpty($packs);
|
||||
$path = $this->root . '/' . $packs[0];
|
||||
$bytes = file_get_contents($path);
|
||||
// Flip a byte well inside the file, away from any header, so decryption's own
|
||||
// authentication tag catches it (this must fail loudly, not just "do nothing").
|
||||
$bytes[(int)(strlen($bytes) / 2)] = chr(~ord($bytes[(int)(strlen($bytes) / 2)]) & 0xFF);
|
||||
file_put_contents($path, $bytes);
|
||||
|
||||
$result = $this->open()->verify($s['snapshot'], deep: true);
|
||||
|
||||
$this->assertNotSame([], $result['failed']);
|
||||
}
|
||||
|
||||
public function testDeepVerifyRetriesAReadThatFailsOnce(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$s = $repo->backupDirectory($this->src);
|
||||
$flaky = new FlakyRangeBackend($this->backend, failures: 1);
|
||||
|
||||
$result = Repository::openWithKey($flaky, $this->keys, null, null)->verify($s['snapshot'], deep: true);
|
||||
|
||||
$this->assertSame(200000, $result['bytesChecked']);
|
||||
$this->assertSame([], $result['failed']);
|
||||
$this->assertSame([], $result['unreadable']);
|
||||
}
|
||||
|
||||
public function testDeepVerifyReportsAPersistentReadErrorAsUnreadableNotCorrupted(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$s = $repo->backupDirectory($this->src);
|
||||
$broken = new FlakyRangeBackend($this->backend, failures: PHP_INT_MAX);
|
||||
|
||||
$result = Repository::openWithKey($broken, $this->keys, null, null)->verify($s['snapshot'], deep: true);
|
||||
|
||||
$this->assertSame([], $result['failed']);
|
||||
$this->assertNotSame([], $result['unreadable']);
|
||||
$this->assertSame('connection reset', $result['readError']);
|
||||
}
|
||||
|
||||
public function testDeepVerifyDoesNotHideAProgrammingError(): void {
|
||||
$repo = Repository::initWithKey($this->backend, $this->keys, $this->keys->wrap('pw'));
|
||||
$s = $repo->backupDirectory($this->src);
|
||||
$buggy = new FlakyRangeBackend($this->backend, failures: PHP_INT_MAX, error: new \TypeError('bug'));
|
||||
|
||||
$this->expectException(\TypeError::class);
|
||||
Repository::openWithKey($buggy, $this->keys, null, null)->verify($s['snapshot'], deep: true);
|
||||
}
|
||||
}
|
||||
|
||||
/** Delegates to a real backend, but getRange() throws for the first $failures calls. */
|
||||
final class FlakyRangeBackend implements IBackend {
|
||||
public function __construct(
|
||||
private IBackend $inner,
|
||||
private int $failures,
|
||||
private ?\Throwable $error = null,
|
||||
) {
|
||||
}
|
||||
|
||||
public function put(string $path, $stream): void {
|
||||
$this->inner->put($path, $stream);
|
||||
}
|
||||
|
||||
public function get(string $path) {
|
||||
return $this->inner->get($path);
|
||||
}
|
||||
|
||||
public function getRange(string $path, int $offset, int $length): string {
|
||||
if ($this->failures > 0) {
|
||||
$this->failures--;
|
||||
throw $this->error ?? new BackendException('connection reset');
|
||||
}
|
||||
return $this->inner->getRange($path, $offset, $length);
|
||||
}
|
||||
|
||||
public function exists(string $path): bool {
|
||||
return $this->inner->exists($path);
|
||||
}
|
||||
|
||||
public function move(string $from, string $to): void {
|
||||
$this->inner->move($from, $to);
|
||||
}
|
||||
|
||||
public function list(string $prefix): array {
|
||||
return $this->inner->list($prefix);
|
||||
}
|
||||
|
||||
public function listFiles(string $dir): array {
|
||||
return $this->inner->listFiles($dir);
|
||||
}
|
||||
|
||||
public function delete(string $path): void {
|
||||
$this->inner->delete($path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
namespace OCA\NgBackup\Tests\Unit\Service;
|
||||
|
||||
use OCA\NgBackup\Service\LeaseService;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
final class LeaseThrottleTest extends TestCase {
|
||||
public function testRefreshesOnlyOncePerInterval(): void {
|
||||
$calls = 0;
|
||||
$beat = LeaseService::throttled(function () use (&$calls): void {
|
||||
$calls++;
|
||||
}, 3600);
|
||||
for ($i = 0; $i < 1000; $i++) {
|
||||
$beat();
|
||||
}
|
||||
$this->assertSame(0, $calls, 'no refresh before the interval has passed');
|
||||
}
|
||||
|
||||
public function testRefreshesEveryCallWithZeroInterval(): void {
|
||||
$calls = 0;
|
||||
$beat = LeaseService::throttled(function () use (&$calls): void {
|
||||
$calls++;
|
||||
}, 0);
|
||||
for ($i = 0; $i < 5; $i++) {
|
||||
$beat();
|
||||
}
|
||||
$this->assertSame(5, $calls);
|
||||
}
|
||||
}
|
||||
@@ -3,3 +3,13 @@
|
||||
declare(strict_types=1);
|
||||
|
||||
require __DIR__ . '/../vendor/autoload.php';
|
||||
|
||||
// nextcloud/ocp ships the public interfaces without an autoloader; tests that mock them need one.
|
||||
spl_autoload_register(static function (string $class): void {
|
||||
if (str_starts_with($class, 'OCP\\')) {
|
||||
$file = __DIR__ . '/../vendor/nextcloud/ocp/' . str_replace('\\', '/', $class) . '.php';
|
||||
if (is_file($file)) {
|
||||
require $file;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
# Disaster recovery: occ backup:key:import + backup:target:add + backup:restore:full on a
|
||||
# SECOND, fresh Nextcloud instance that never saw the first one, sharing only the backup
|
||||
# location (a local directory, bind-mounted into both containers) -- see PLAN.md 3.7.
|
||||
# Usage: full-restore-roundtrip.sh <nextcloud-image-tag>
|
||||
# e.g. full-restore-roundtrip.sh 35-apache
|
||||
set -euo pipefail
|
||||
IMAGE="nextcloud:${1:-35-apache}"
|
||||
APP=$(cd "$(dirname "$0")/../.." && pwd)
|
||||
NET=ngb-frt-net
|
||||
SHARED=$(mktemp -d)
|
||||
chmod 777 "$SHARED"
|
||||
fail() { echo "FAIL: $*"; exit 1; }
|
||||
ok() { echo "ok $*"; }
|
||||
|
||||
cleanup() {
|
||||
docker rm -fv ngb-frt-origin ngb-frt-restore >/dev/null 2>&1 || true
|
||||
docker network rm "$NET" >/dev/null 2>&1 || true
|
||||
docker run --rm -v "$SHARED:/target" alpine sh -c 'rm -rf /target/*' >/dev/null 2>&1 || true
|
||||
rmdir "$SHARED" 2>/dev/null || true
|
||||
rm -f /tmp/ngb-frt-kit.json /tmp/ngb-frt-pass
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
run_on() { docker exec -u www-data "$1" php occ "${@:2}"; }
|
||||
|
||||
docker network create "$NET" >/dev/null
|
||||
docker run -d --name ngb-frt-origin --network "$NET" -e SQLITE_DATABASE=nextcloud \
|
||||
-e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \
|
||||
-v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null
|
||||
for i in $(seq 1 90); do run_on ngb-frt-origin status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done
|
||||
docker exec ngb-frt-origin chown www-data:www-data /var/www/html/custom_apps
|
||||
run_on ngb-frt-origin app:enable ng_backup >/dev/null
|
||||
ok "origin instance up"
|
||||
|
||||
docker exec ngb-frt-origin sh -c "printf 'full restore roundtrip passphrase 2026\n' > /tmp/pass"
|
||||
run_on ngb-frt-origin backup:key:init --passphrase-file=/tmp/pass --label=CI >/dev/null || fail "key init"
|
||||
run_on ngb-frt-origin backup:key:kit --output=/tmp/kit.json >/dev/null
|
||||
CODE=$(run_on ngb-frt-origin backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];')
|
||||
run_on ngb-frt-origin backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "confirm"
|
||||
run_on ngb-frt-origin backup:target:add offsite local -a null::null -o datadir=/mnt/shared --path=repo >/dev/null || fail "target add"
|
||||
ok "key, kit and location on the origin"
|
||||
|
||||
NC_PASS=$(openssl rand -hex 16)
|
||||
docker exec -u www-data -e OC_PASS="$NC_PASS" ngb-frt-origin php occ user:add --password-from-env frtest >/dev/null
|
||||
docker exec ngb-frt-origin sh -c "mkdir -p /var/www/html/data/frtest/files/Docs && head -c 200000 /dev/urandom > /var/www/html/data/frtest/files/Docs/report.bin && echo 'hello disaster recovery' > /var/www/html/data/frtest/files/Docs/note.txt && chown -R www-data:www-data /var/www/html/data/frtest"
|
||||
run_on ngb-frt-origin files:scan frtest >/dev/null
|
||||
RUN_OUT=$(run_on ngb-frt-origin backup:run offsite -l disaster-test)
|
||||
SNAP=$(printf '%s\n' "$RUN_OUT" | head -1 | awk -F'snapshot ' '{print $2}' | tr -d ',')
|
||||
[ -n "$SNAP" ] || fail "no snapshot id from backup:run"
|
||||
ok "test data backed up, snapshot $SNAP"
|
||||
|
||||
docker cp ngb-frt-origin:/tmp/kit.json /tmp/ngb-frt-kit.json
|
||||
docker exec ngb-frt-origin sh -c "cat /tmp/pass" > /tmp/ngb-frt-pass
|
||||
docker run -d --name ngb-frt-restore --network "$NET" -e SQLITE_DATABASE=nextcloud \
|
||||
-e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin \
|
||||
-v "$APP:/var/www/html/custom_apps/ng_backup:ro" -v "$SHARED:/mnt/shared" "$IMAGE" >/dev/null
|
||||
for i in $(seq 1 90); do run_on ngb-frt-restore status 2>/dev/null | grep -q "installed: true" && break; sleep 3; done
|
||||
docker exec ngb-frt-restore chown www-data:www-data /var/www/html/custom_apps
|
||||
run_on ngb-frt-restore app:enable ng_backup >/dev/null
|
||||
docker cp /tmp/ngb-frt-kit.json ngb-frt-restore:/tmp/kit.json
|
||||
docker exec ngb-frt-restore chmod 644 /tmp/kit.json
|
||||
docker cp /tmp/ngb-frt-pass ngb-frt-restore:/tmp/pass
|
||||
ok "fresh restore instance up"
|
||||
|
||||
run_on ngb-frt-restore backup:key:import /tmp/kit.json --passphrase-file=/tmp/pass >/dev/null || fail "key import"
|
||||
run_on ngb-frt-restore backup:target:add offsite local -a null::null -o datadir=/mnt/shared --path=repo >/dev/null || fail "target add on the restore instance"
|
||||
run_on ngb-frt-restore backup:list offsite | grep -q "$SNAP" || fail "the origin's snapshot is not listed after connecting the location"
|
||||
ok "recovery kit imported, existing repository adopted, snapshot visible"
|
||||
|
||||
run_on ngb-frt-restore backup:restore:full offsite "$SNAP" --no-interaction >/dev/null || fail "restore:full"
|
||||
ok "backup:restore:full completed"
|
||||
|
||||
# Everything below must work in a FRESH occ process (a new PHP process per docker exec), since
|
||||
# earlier bugs here only showed up after config.php's secret was rewritten mid-restore.
|
||||
run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after restore (own key/target undecryptable?)"
|
||||
run_on ngb-frt-restore user:list | grep -q frtest || fail "the backed-up user is missing from the restored database"
|
||||
docker exec ngb-frt-restore test -f /var/www/html/data/frtest/files/Docs/note.txt || fail "restored file is missing"
|
||||
[ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt)" = "hello disaster recovery" ] || fail "restored file content differs"
|
||||
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore"
|
||||
ok "database, files and ng_backup's own state are all intact and usable after the restore"
|
||||
|
||||
run_on ngb-frt-restore backup:run offsite -l post-restore-check >/dev/null || fail "a new backup after the restore fails"
|
||||
ok "the restored instance can run a new backup of its own"
|
||||
|
||||
echo "FULL RESTORE ROUNDTRIP PASS"
|
||||
@@ -49,6 +49,11 @@ SUM_AFTER=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xa
|
||||
$X sh -c "ls $DATA/$U/files_trashbin/files/ | grep -q '^extra.txt'" || fail "extra file not in trash"
|
||||
ok "restore new-folder and replace: files identical, extra file in the trash"
|
||||
|
||||
# Checksum audit: shallow (index presence) and deep (download, decrypt, verify every blob).
|
||||
occ backup:verify ci "$SNAP" | grep -q "^OK$" || fail "shallow verify"
|
||||
occ backup:verify ci "$SNAP" --deep | grep -q "^OK$" || fail "deep verify"
|
||||
ok "shallow and deep checksum audit"
|
||||
|
||||
# Database dump/restore round trip (all tables, checksums, sequences on PostgreSQL).
|
||||
$X php -d memory_limit=512M "$APPS/ng_backup/spikes/03-db-dump.php" /tmp/ngb-rt-db | grep "ALL PASS" >/dev/null || fail "database round trip"
|
||||
ok "database dump and restore"
|
||||
|
||||
@@ -1,17 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Round trip of NG Backup against real S3 (s3mock) and SFTP backends, through files_external's own
|
||||
# backend/auth classes, same as a production location would use. Backend correctness does not
|
||||
# depend on the Nextcloud version or database, so this runs once (not matrixed across NC/DB),
|
||||
# on a recent Nextcloud with sqlite. Each backend gets its own fresh Nextcloud container, since
|
||||
# NG Backup's key can only be initialised once per instance.
|
||||
# Round trip of NG Backup against real S3 (s3mock), SFTP, FTP and SMB backends, through
|
||||
# files_external's own backend/auth classes, same as a production location would use. Backend
|
||||
# correctness does not depend on the Nextcloud version or database, so this runs once (not
|
||||
# matrixed across NC/DB), on a recent Nextcloud with sqlite. Each backend gets its own fresh
|
||||
# Nextcloud container, since NG Backup's key can only be initialised once per instance.
|
||||
# Usage: tests/integration/targets-matrix.sh [nextcloud-version]
|
||||
set -euo pipefail
|
||||
V=${1:-35}
|
||||
APP=$(cd "$(dirname "$0")/../.." && pwd)
|
||||
NET=ngb-targets-net
|
||||
SMB_IMAGE=ngb-targets-smb-nc
|
||||
SMB_BUILD_DIR=$(mktemp -d)
|
||||
RESULTS=()
|
||||
|
||||
cleanup() { docker rm -fv ngb-targets-nc-s3 ngb-targets-nc-sftp ngb-targets-s3 ngb-targets-sftp >/dev/null 2>&1; docker network rm "$NET" >/dev/null 2>&1; }
|
||||
cleanup() {
|
||||
docker rm -fv ngb-targets-nc-s3 ngb-targets-nc-sftp ngb-targets-nc-ftp ngb-targets-nc-smb \
|
||||
ngb-targets-s3 ngb-targets-sftp ngb-targets-ftp ngb-targets-smb >/dev/null 2>&1 || true
|
||||
docker rmi "$SMB_IMAGE" >/dev/null 2>&1 || true
|
||||
docker network rm "$NET" >/dev/null 2>&1 || true
|
||||
rm -rf "$SMB_BUILD_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
wait_tcp() { # host port
|
||||
@@ -19,11 +27,11 @@ wait_tcp() { # host port
|
||||
|| { echo "FAIL: $1:$2 never came up"; exit 1; }
|
||||
}
|
||||
|
||||
start_nc() { # container-name
|
||||
start_nc() { # container-name [image]
|
||||
docker run -d --name "$1" --network "$NET" \
|
||||
-e NEXTCLOUD_ADMIN_USER=admin -e NEXTCLOUD_ADMIN_PASSWORD=admin-targets-pw -e NEXTCLOUD_TRUSTED_DOMAINS=localhost \
|
||||
-e SQLITE_DATABASE=nextcloud \
|
||||
-v "$APP:/var/www/html/custom_apps/ng_backup:ro" nextcloud:$V-apache >/dev/null
|
||||
-v "$APP:/var/www/html/custom_apps/ng_backup:ro" "${2:-nextcloud:$V-apache}" >/dev/null
|
||||
for i in $(seq 1 90); do docker exec -u www-data "$1" php occ status 2>/dev/null | grep -q "installed: true" && break; sleep 4; done
|
||||
docker exec "$1" chown www-data:www-data /var/www/html/custom_apps
|
||||
echo "=== NC $V / sqlite ($1): $(docker exec -u www-data "$1" php occ -V 2>/dev/null)"
|
||||
@@ -51,8 +59,37 @@ docker run -d --name ngb-targets-s3 --network "$NET" \
|
||||
docker run -d --name ngb-targets-sftp --network "$NET" \
|
||||
atmoz/sftp ngbackup:ngbackuppw:::upload >/dev/null
|
||||
|
||||
# stilliard/pure-ftpd: delfer/alpine-ftp-server (vsftpd) was tried first and rejects the absolute
|
||||
# paths files_external's FTP backend always builds, inside this server's chroot specifically --
|
||||
# not a files_external bug (confirmed with plain php ftp_mkdir()), but pure-ftpd sidesteps it.
|
||||
docker run -d --name ngb-targets-ftp --network "$NET" \
|
||||
-e "FTP_USER_NAME=ngbackup" -e "FTP_USER_PASS=ngbackuppw" -e "FTP_USER_HOME=/home/ngbackup" \
|
||||
-e "PUBLICHOST=ngb-targets-ftp" \
|
||||
stilliard/pure-ftpd >/dev/null
|
||||
|
||||
# dperson/samba. The *native* smbclient PECL extension is required: files_external's shell-based
|
||||
# fallback (just the smbclient CLI binary) still registers this backend, but its stream cannot
|
||||
# seek, and ng_backup reads blobs back by byte range while backing up -- confirmed live, that
|
||||
# fallback fails the very first (small) backup with "Seek failed in packs/...". The extension is
|
||||
# not in the stock nextcloud image (needs libsmbclient-dev + a compile), so this backend gets its
|
||||
# own small image, built once here, in its own directory (never /tmp directly: that would send
|
||||
# every unrelated file under /tmp to the docker daemon as build context).
|
||||
cat > "$SMB_BUILD_DIR/Dockerfile" <<EOF
|
||||
FROM nextcloud:$V-apache
|
||||
RUN apt-get update -qq && apt-get install -y -qq libsmbclient-dev >/dev/null \\
|
||||
&& pecl install smbclient >/dev/null 2>&1 \\
|
||||
&& docker-php-ext-enable smbclient \\
|
||||
&& apt-get clean
|
||||
EOF
|
||||
docker build -q -t "$SMB_IMAGE" "$SMB_BUILD_DIR" >/dev/null
|
||||
docker run -d --name ngb-targets-smb --network "$NET" \
|
||||
-e USERID=1000 -e GROUPID=1000 \
|
||||
dperson/samba -p -u "ngbackup;ngbackuppw" -s "backup;/share;yes;no;no;ngbackup" >/dev/null
|
||||
|
||||
wait_tcp ngb-targets-s3 9090
|
||||
wait_tcp ngb-targets-sftp 22
|
||||
wait_tcp ngb-targets-ftp 21
|
||||
wait_tcp ngb-targets-smb 445
|
||||
|
||||
start_nc ngb-targets-nc-s3
|
||||
run "S3 (s3mock)" ngb-targets-nc-s3 "amazons3 -a amazons3::accesskey -o bucket=ngbackups -o hostname=ngb-targets-s3 -o port=9090 -o region=us-east-1 -o use_ssl=false -o use_path_style=true -o key=test -o secret=test"
|
||||
@@ -60,6 +97,12 @@ run "S3 (s3mock)" ngb-targets-nc-s3 "amazons3 -a amazons3::accesskey -o bucket=n
|
||||
start_nc ngb-targets-nc-sftp
|
||||
run "SFTP" ngb-targets-nc-sftp "sftp -a password::password -o host=ngb-targets-sftp -o root=/upload -o user=ngbackup -o password=ngbackuppw"
|
||||
|
||||
start_nc ngb-targets-nc-ftp
|
||||
run "FTP" ngb-targets-nc-ftp "ftp -a password::password -o host=ngb-targets-ftp -o user=ngbackup -o password=ngbackuppw"
|
||||
|
||||
start_nc ngb-targets-nc-smb "$SMB_IMAGE"
|
||||
run "SMB" ngb-targets-nc-smb "smb -a password::password -o host=ngb-targets-smb -o share=backup -o user=ngbackup -o password=ngbackuppw"
|
||||
|
||||
printf '%s\n' "${RESULTS[@]}"
|
||||
printf '%s\n' "${RESULTS[@]}" | grep -q FAIL && exit 1
|
||||
exit 0
|
||||
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/bin/bash
|
||||
# Per-user backup/restore round trip via user_migration (optional dependency, see PLAN.md 3.8):
|
||||
# occ backup:user:backup / occ backup:user:restore --mode=as-backup|replace.
|
||||
# Usage: user-roundtrip.sh "<command prefix to run occ/php as the web user>" <datadir> [target-args]
|
||||
# e.g. user-roundtrip.sh "docker exec -u www-data ngb-nc35-pgsql" /var/www/html/data
|
||||
set -euo pipefail
|
||||
X="$1"; DATA="$2"
|
||||
TARGET_ARGS="${3:-local -a null::null -o datadir=/tmp/ngb-urt-target}"
|
||||
occ() { $X php occ "$@"; }
|
||||
fail() { echo "FAIL: $*"; exit 1; }
|
||||
ok() { echo "ok $*"; }
|
||||
U=ngbutest-$RANDOM
|
||||
|
||||
occ app:enable ng_backup >/dev/null
|
||||
if ! occ app:install user_migration >/dev/null 2>&1 && ! occ app:enable user_migration >/dev/null 2>&1; then
|
||||
echo "SKIP: user_migration is not available (no network, or the app store is unreachable) -- per-user backup/restore not tested"
|
||||
exit 0
|
||||
fi
|
||||
sleep 8 # APCu app config cache
|
||||
|
||||
$X sh -c "printf 'user roundtrip passphrase 2026\n' > /tmp/ngb-urt-pass && mkdir -p /tmp/ngb-urt-target"
|
||||
occ backup:key:init --passphrase-file=/tmp/ngb-urt-pass --label=CI --delete-delay-days=7 >/dev/null || fail "key init"
|
||||
CODE=$(occ backup:key:kit | php -r 'echo json_decode(stream_get_contents(STDIN), true)["confirmation_code"];')
|
||||
occ backup:key:confirm -u admin --code="$CODE" --confirm="Yes, I confirm" >/dev/null || fail "confirm"
|
||||
ok "key, kit code and confirmation"
|
||||
|
||||
occ backup:target:add ci $TARGET_ARGS --path=repo >/dev/null || fail "target add"
|
||||
ok "location added"
|
||||
|
||||
$X sh -c "OC_PASS='User-roundtrip-pw-2026' php occ user:add --password-from-env $U >/dev/null"
|
||||
$X sh -c "mkdir -p $DATA/$U/files/Docs && for i in 1 2 3; do head -c 100000 /dev/urandom > $DATA/$U/files/Docs/f\$i.bin; done"
|
||||
occ files:scan "$U" >/dev/null
|
||||
SUM_BEFORE=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
|
||||
|
||||
MANIFEST=$(occ backup:user:backup ci "$U" | sed -n 's/.* to \(.*\)\.$/\1/p')
|
||||
[ -n "$MANIFEST" ] || fail "backup:user:backup printed no manifest path"
|
||||
ok "backed up $U to $MANIFEST"
|
||||
|
||||
# as-backup: restore into <uid>-bak; the original account must stay untouched.
|
||||
occ backup:user:restore ci "$MANIFEST" --mode=as-backup >/dev/null || fail "restore as-backup"
|
||||
occ files:scan "$U-bak" >/dev/null
|
||||
SUM_BAK=$($X sh -c "cd $DATA/$U-bak/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
|
||||
[ "$SUM_BEFORE" = "$SUM_BAK" ] || fail "the <uid>-bak account's files differ from the original"
|
||||
occ user:delete "$U-bak" >/dev/null
|
||||
$X sh -c "[ -d '$DATA/$U/files/Docs' ]" || fail "the original account should be untouched by an as-backup restore"
|
||||
ok "as-backup restores into <uid>-bak, byte-identical, original account untouched"
|
||||
|
||||
# replace: damage the account, then restore it in place under the same uid.
|
||||
$X sh -c "rm '$DATA/$U/files/Docs/f1.bin' && echo damaged > '$DATA/$U/files/Docs/f1.bin'"
|
||||
occ files:scan "$U" >/dev/null
|
||||
if occ backup:user:restore ci "$MANIFEST" --mode=replace --no-interaction >/dev/null 2>&1; then fail "replace of an existing account ran without --force"; fi
|
||||
occ backup:user:restore ci "$MANIFEST" --mode=replace --force >/dev/null || fail "restore replace"
|
||||
occ files:scan "$U" >/dev/null
|
||||
SUM_AFTER=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
|
||||
[ "$SUM_BEFORE" = "$SUM_AFTER" ] || fail "files differ after a replace restore"
|
||||
ok "replace restores the original content under the same uid, byte-identical"
|
||||
|
||||
# Retention: $U now has the manual export above plus the automatic safety-export the replace
|
||||
# restore made of it just before deleting it (4 total after these two more), then --user-last=2
|
||||
# must forget exactly the 2 oldest.
|
||||
sleep 1.1 # exportId is time-based to the second; force each one to be distinct
|
||||
occ backup:user:backup ci "$U" >/dev/null || fail "third backup:user:backup"
|
||||
sleep 1.1
|
||||
occ backup:user:backup ci "$U" >/dev/null || fail "fourth backup:user:backup"
|
||||
occ backup:retention --user-last=2 >/dev/null || fail "retention --user-last"
|
||||
PRUNE_OUT=$(occ backup:prune ci)
|
||||
echo "$PRUNE_OUT" | grep -q "forgot 2 user export(s) for 1 user(s)" || fail "prune did not forget the expected two user exports"
|
||||
ok "user-export retention forgets exactly the exports over the keep count"
|
||||
|
||||
occ user:delete "$U" >/dev/null
|
||||
$X sh -c "rm -rf /tmp/ngb-urt-target /tmp/ngb-urt-pass"
|
||||
echo "USER ROUNDTRIP PASS"
|
||||
Reference in New Issue
Block a user