Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e5eaf6df14 | |||
| dad60140bd | |||
| b144f20453 | |||
| 50acd5dc0e | |||
| 2ab4b2e7d5 | |||
| 9cefea019d | |||
| b42acb41c5 | |||
| 4a13f8e791 | |||
| 7b00d7ab94 | |||
| aa541d58b2 | |||
| e00d6579f7 | |||
| 101f954c5d | |||
| 6ee2b7e138 | |||
| b06ff0f89f | |||
| 5b0aded7b0 | |||
| 4fce893389 | |||
| fee189aec3 |
@@ -1,106 +0,0 @@
|
|||||||
# SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
name: CI
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
pull_request:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
unit:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
php: ['8.2', '8.3', '8.4']
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
- uses: shivammathur/setup-php@v2
|
|
||||||
with:
|
|
||||||
php-version: ${{ matrix.php }}
|
|
||||||
extensions: sodium, zip
|
|
||||||
coverage: none
|
|
||||||
- run: composer install --no-interaction --no-progress
|
|
||||||
- run: vendor/bin/phpunit -c tests/phpunit.xml
|
|
||||||
|
|
||||||
psalm:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
- uses: shivammathur/setup-php@v2
|
|
||||||
with:
|
|
||||||
php-version: '8.2'
|
|
||||||
extensions: sodium, zip
|
|
||||||
coverage: none
|
|
||||||
- run: composer install --no-interaction --no-progress
|
|
||||||
- run: composer psalm
|
|
||||||
|
|
||||||
integration:
|
|
||||||
# Full round trip (key, kit, location, backup, incremental, restore modes, database dump and
|
|
||||||
# restore, retention, trash) on a fresh Nextcloud per version and database.
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
nextcloud: ['32', '33', '34', '35']
|
|
||||||
database: ['sqlite', 'mariadb', 'pgsql']
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
- name: Round trip on Nextcloud ${{ matrix.nextcloud }} / ${{ matrix.database }}
|
|
||||||
run: tests/integration/matrix.sh "${{ matrix.nextcloud }}" "${{ matrix.database }}"
|
|
||||||
|
|
||||||
integration-targets:
|
|
||||||
# Same round trip, but through the real S3 (s3mock) and SFTP backends instead of a local
|
|
||||||
# path, so files_external's AmazonS3/SFTP backend and auth classes are actually exercised.
|
|
||||||
# Backend correctness does not depend on the Nextcloud version or database, so this runs once.
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
- name: Round trip against S3 and SFTP
|
|
||||||
run: tests/integration/targets-matrix.sh
|
|
||||||
|
|
||||||
integration-user:
|
|
||||||
# Per-user backup/restore via user_migration (occ backup:user:backup / backup:user:restore),
|
|
||||||
# an optional dependency installed from the app store. It calls user_migration's internal
|
|
||||||
# UserMigrationService, whose API can change between Nextcloud versions, so this runs on every
|
|
||||||
# supported version (not the database, which the main integration job already covers).
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
matrix:
|
|
||||||
nextcloud: ['32', '33', '34', '35']
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
- name: Set up a fresh Nextcloud instance
|
|
||||||
run: |
|
|
||||||
docker run -d --name ngbu -e SQLITE_DATABASE=nextcloud -e NEXTCLOUD_ADMIN_USER=admin \
|
|
||||||
-e NEXTCLOUD_ADMIN_PASSWORD=admin-user-roundtrip -e NEXTCLOUD_TRUSTED_DOMAINS=localhost \
|
|
||||||
-v "$PWD:/var/www/html/custom_apps/ng_backup:ro" nextcloud:${{ matrix.nextcloud }}-apache
|
|
||||||
for i in $(seq 1 90); do docker exec -u www-data ngbu php occ status 2>/dev/null | grep -q "installed: true" && break; sleep 4; done
|
|
||||||
docker exec ngbu chown www-data:www-data /var/www/html/custom_apps
|
|
||||||
- name: Per-user backup/restore round trip on Nextcloud ${{ matrix.nextcloud }}
|
|
||||||
run: tests/integration/user-roundtrip.sh "docker exec -u www-data ngbu" /var/www/html/data
|
|
||||||
|
|
||||||
integration-full-restore:
|
|
||||||
# Disaster recovery (occ backup:restore:full) on a SECOND, fresh Nextcloud instance that
|
|
||||||
# never saw the first one, sharing only the backup location. Does not depend on the database
|
|
||||||
# beyond what the main integration job already covers, so this runs once, on one NC version.
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
- name: Full restore round trip
|
|
||||||
run: tests/integration/full-restore-roundtrip.sh 35-apache
|
|
||||||
@@ -1,10 +1,7 @@
|
|||||||
/build/
|
/build/
|
||||||
cred-*
|
|
||||||
*credentials*
|
|
||||||
/css/
|
/css/
|
||||||
/js/
|
/js/
|
||||||
/node_modules/
|
/node_modules/
|
||||||
.phpunit.result.cache
|
.phpunit.result.cache
|
||||||
*.secret
|
|
||||||
/spikes/out/
|
/spikes/out/
|
||||||
/vendor/
|
/vendor/
|
||||||
|
|||||||
-255
@@ -1,255 +0,0 @@
|
|||||||
# Changelog
|
|
||||||
|
|
||||||
All notable changes to NG Backup are documented here. The format follows
|
|
||||||
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows the stages in
|
|
||||||
`PLAN.md` section 9. The `<version>` in `appinfo/info.xml` is plain SemVer-numeric on purpose:
|
|
||||||
the app store accepts a `-beta` suffix, but Nextcloud servers then only offer the release on the
|
|
||||||
beta/daily update channel, which would hide the app from most admins. The stability stage is
|
|
||||||
instead carried by the app description, the git tag, the release, and this file.
|
|
||||||
|
|
||||||
## [0.9.6-beta.7] - 2026-10-14
|
|
||||||
|
|
||||||
Bugfix release for SMB locations, with `occ backup:cancel` to stop a backup that does not get
|
|
||||||
anywhere. Tested on a production instance: a first full backup of 151,021 files (807 GiB) to OVH
|
|
||||||
S3 completed, and a backup to a Hetzner Storage Box over SMB that had slowed to 12 GiB in 35
|
|
||||||
hours ran at about 5.6 MiB/s after the update, while the S3 backup was still reading.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- The list of backup runs (web interface and `occ backup:status`) shows the total size of each
|
|
||||||
backup next to what it uploaded: an incremental backup of 1 TB that uploads 800 MB now shows
|
|
||||||
both. `occ backup:status` shows sizes in GiB where that reads better than MiB.
|
|
||||||
- `occ backup:cancel <run>` stops a running backup; `occ backup:status` shows the run numbers.
|
|
||||||
No new step of the run starts, the step in progress finishes (usually within a minute), then
|
|
||||||
the process working on the run (`occ backup:run` or cron) stops. What it already stored is
|
|
||||||
removed by the next cleanup. A backup started by an earlier version is stopped as well: after
|
|
||||||
two minutes it is marked cancelled while it works, and its process stops at the end of the
|
|
||||||
step.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Backups to SMB locations, and to other slow locations with many packs, slowed to a crawl: each
|
|
||||||
30-second step opened the location again, and reading the list of index files took longer
|
|
||||||
than the step itself. Each step then stored a single 4 MiB pack, which made the list longer
|
|
||||||
still. A first full backup of 0.8 TB to a Hetzner Storage Box over SMB managed 12 GiB in 35
|
|
||||||
hours. `occ backup:run` and system cron now open the location once per run, opening no longer
|
|
||||||
counts towards the step's time, and SMB folders are listed in one request instead of one per
|
|
||||||
file. A backup to an SMB location that is not getting anywhere keeps running in its old
|
|
||||||
process after the update: update first, then stop it with `occ backup:cancel <run>` and start
|
|
||||||
it again. The new backup reuses the packs the old one stored.
|
|
||||||
|
|
||||||
## [0.9.5-beta.6] - 2026-10-12
|
|
||||||
|
|
||||||
Bugfix release for SFTP locations and large repositories. Tested on a production instance with a
|
|
||||||
1 TB repository on a Hetzner Storage Box over SFTP: a check in 5.5 minutes, a deep check of all
|
|
||||||
316,529 blobs (815 GiB) in 11 hours, and an incremental backup in under 8 minutes.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Progress of a running backup in the web interface and `occ backup:status`, e.g.
|
|
||||||
"running (files) 68%". The total is the size of the previous snapshot; for a first backup it
|
|
||||||
is estimated from Nextcloud's file cache (users' files, versions and trash) and shown as
|
|
||||||
"~68%".
|
|
||||||
- Edit a backup location (web interface and `occ backup:target:edit`): change the password, host,
|
|
||||||
folder, connection type or name without losing its history. Before this the only option was a
|
|
||||||
new location, which starts an empty repository and, with a retention lock at the provider,
|
|
||||||
keeps paying for the old one. The new settings are only saved when they lead to the same
|
|
||||||
repository; stored passwords are never sent to the browser, and an empty password field keeps
|
|
||||||
the stored one. Administrators are notified of the change.
|
|
||||||
- `occ backup:verify` reports how far it is every 30 seconds, and `occ backup:status` shows the
|
|
||||||
progress of a running check and how the last check of each location ended. On a large
|
|
||||||
repository a check takes hours, a deep one days.
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- Checking and opening a large repository over SFTP is much faster: the index files (one per
|
|
||||||
pack, 27,547 for 1 TB) are downloaded 32 at a time instead of one after the other, 114 instead
|
|
||||||
of 6 files per second to a Hetzner Storage Box (about 4 minutes instead of 77 for 1 TB). A check
|
|
||||||
of a repository with more than 256 packs also lists the pack folders (256 requests) instead of
|
|
||||||
asking for every pack.
|
|
||||||
- App store listing: the author's e-mail address is shown.
|
|
||||||
- App store listing: the homepage link points to the GitHub repository instead of back to the
|
|
||||||
app store page itself.
|
|
||||||
- The notification for a location without a successful backup read "No successful backup to …
|
|
||||||
since never" when there had never been one; it now reads "No successful backup to … yet".
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- 0.9.4 could not open a repository at the root of an SFTP location (empty path): its faster
|
|
||||||
SFTP reads sent their first request before phpseclib had opened the SFTP channel, which only
|
|
||||||
happens on the first SFTP call after login. Every backup then failed with "No repository at
|
|
||||||
this location". The channel is now opened first.
|
|
||||||
- A check (`occ backup:verify`) did not renew its hold on the location while it downloaded the
|
|
||||||
index; on a large repository that takes longer than the hold lasts, so a weekly cleanup could
|
|
||||||
have started during the check.
|
|
||||||
- A long first backup could end as failed although its snapshot was complete: when its final
|
|
||||||
step took longer than the step's lease (150 s with `occ backup:run`), the background job took
|
|
||||||
the run over, finished it a second time and failed on the snapshot it had already recorded,
|
|
||||||
overwriting the result. The lease now lasts at least 15 minutes, a step writes the run only if
|
|
||||||
no other process wrote it since the step loaded it (in one database statement), and recording
|
|
||||||
a snapshot that is already recorded is no longer an error.
|
|
||||||
- `occ backup:verify` on a large repository failed with "SFTP connection failed while opening
|
|
||||||
snapshots/…" or "Not found: snapshots/…". A Hetzner Storage Box drops an SFTP connection that
|
|
||||||
carried a lot of data (some 40 MB) and then stayed quiet for a few minutes, and the check went
|
|
||||||
quiet after downloading the index: it refreshed the local index cache, first in 29 minutes (a
|
|
||||||
pass over all blobs for every pack: 27,547 packs, 400,000 blobs for 1 TB), then still in more
|
|
||||||
than 6 minutes (a database transaction per pack on PostgreSQL). Nextcloud's SFTP storage keeps
|
|
||||||
using a dropped connection, on which every file looks missing. Now:
|
|
||||||
- an SFTP connection that was not used for a minute is checked first, and one that turns out
|
|
||||||
to be dead after a failed request is replaced, once per request; uploads, moves and deletes
|
|
||||||
are only checked first, never repeated;
|
|
||||||
- the index cache is written in one transaction (27,547 packs in 11 seconds on a test server,
|
|
||||||
against about a minute row by row);
|
|
||||||
- after a check the cache holds the index files as they are: before, a blob that was in two
|
|
||||||
packs was only cached for one of them.
|
|
||||||
|
|
||||||
## [0.9.4-beta.5] - 2026-10-09
|
|
||||||
|
|
||||||
Bugfix release: faster restores.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Restores and `occ backup:verify --deep` fetched every blob with its own request, so on a
|
|
||||||
remote location they ran far below the speed of the connection. The blobs of a pack are now
|
|
||||||
read from one open stream, in the order they were written. Restoring 86 MB in 202 files from a
|
|
||||||
Hetzner Storage Box over SMB went from 0.78 MB/s to 4.2 MB/s, the speed of reading the packs
|
|
||||||
as plain files over that connection.
|
|
||||||
- Reading from an SFTP location waited a full round trip for every 256 KiB, whatever the line
|
|
||||||
speed. Reads now keep up to 8 requests in flight. Restoring 1.97 GB in 500 files from a Hetzner
|
|
||||||
Storage Box (47 ms away) took 694 s before the fix above, 499 s with it and 142 s with both
|
|
||||||
(2.8 → 3.9 → 13.9 MB/s); a full 51 GB restore ran at 13.0 MB/s. When a server error
|
|
||||||
interrupts such a read, it goes on from the same position with Nextcloud's own SFTP stream;
|
|
||||||
after three failing files in a row, or on a Nextcloud whose SFTP library lacks what this needs,
|
|
||||||
only Nextcloud's own stream is used.
|
|
||||||
- README: a WebDAV location on a slow upload needs a higher `davstorage.request_timeout`.
|
|
||||||
|
|
||||||
## [0.9.3-beta.4] - 2026-10-08
|
|
||||||
|
|
||||||
Bugfix release.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- A backup failed with "cannot execute INSERT in a read-only transaction" on PostgreSQL as soon
|
|
||||||
as the database dump filled a whole pack (32 MiB): the pack's index cache row was written
|
|
||||||
inside the read-only dump transaction. On other databases that row was rolled back silently.
|
|
||||||
- Backups to a remote location slowed down as the repository grew: every backup step listed the
|
|
||||||
repository's index with a round trip per file. Over SFTP to a Hetzner Storage Box a 60 GiB
|
|
||||||
repository already lost a large part of each step to it; backing up continuously
|
|
||||||
(`occ backup:run`) went from 45 KiB/s to 5.8 MB/s with the fix.
|
|
||||||
|
|
||||||
## [0.9.2-beta.3] - 2026-10-08
|
|
||||||
|
|
||||||
Bugfix release.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- README: notes on using a Hetzner Storage Box over WebDAV and SMB.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Adding a backup location in the web interface failed with "Request failed with status code 403"
|
|
||||||
when the login was more than 30 minutes old: the form did not ask for password confirmation.
|
|
||||||
- Settings page: the "Backup and restore" / "Settings" tabs line up with the sections below them.
|
|
||||||
- Backups: the runs table scrolls horizontally on narrow screens instead of running off the page.
|
|
||||||
- `occ backup:verify --deep`: a blob that could not be read (network or storage error) was reported
|
|
||||||
as "corrupted or tampered". Reads are now retried, and a read that keeps failing is reported
|
|
||||||
separately as unreadable, with the error (run the check again; if the same blobs stay unreadable,
|
|
||||||
their pack may be damaged or truncated). Only a blob that was read but fails decryption or its
|
|
||||||
hash check counts as corrupted. Found on a Hetzner Storage Box over WebDAV, where the blob
|
|
||||||
turned out to be intact.
|
|
||||||
|
|
||||||
## [0.9.1-beta.2] - 2026-10-07
|
|
||||||
|
|
||||||
Second beta, the first release in the Nextcloud app store (signed).
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- App store listing: screenshots, website, repository and documentation link in `appinfo/info.xml`.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Restore: the "new folder" option showed `"Restored "` without the date pattern.
|
|
||||||
- Restore: the restore-point dropdown cut off the time.
|
|
||||||
- Settings: the passphrase actions are on one line instead of stacked.
|
|
||||||
- `docs/HOWTO.md`: the SFTP example used a non-existent auth mechanism (`sftp::password`
|
|
||||||
instead of `password::password`).
|
|
||||||
|
|
||||||
## [0.9.0-beta.1] - 2026-10-07
|
|
||||||
|
|
||||||
First beta. Tested thoroughly and usable **alongside** your current backup, but **not yet a
|
|
||||||
drop-in replacement** for it. We test what we can ourselves (Nextcloud 32–35, SQLite, MariaDB and
|
|
||||||
PostgreSQL, S3, SFTP, FTP, SMB and NFS), but we need field experience from other setups: please
|
|
||||||
report what works and what doesn't, especially on large instances and on storage providers not
|
|
||||||
listed in the README.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Disaster recovery: `occ backup:key:import` and `occ backup:restore:full` restore the database,
|
|
||||||
data directory and config onto a fresh installation of the same major version.
|
|
||||||
- Per-user backup and restore via user_migration: `occ backup:user:backup` and
|
|
||||||
`occ backup:user:restore` (replace the account after a safety export, or restore as
|
|
||||||
`<uid>-bak`), also onto a server where the user does not exist yet. The commands ask before
|
|
||||||
installing user_migration (or take `--install-user-migration`).
|
|
||||||
- Checksum audit: `occ backup:verify`, shallow or `--deep`, for snapshots and per-user exports.
|
|
||||||
- Retention for per-user exports: `occ backup:retention --user-last=N`.
|
|
||||||
- FTP and SMB covered by the integration tests; NFS through the `local` backend on a mounted share.
|
|
||||||
- `docs/HOWTO.md`: step by step from setup to a tested restore.
|
|
||||||
- Warnings when a backup finishes with unreadable files or folders, or with files that kept
|
|
||||||
changing while being read (log, `occ`, notification, snapshot).
|
|
||||||
|
|
||||||
### Changed
|
|
||||||
- `backup:restore:full` refuses on an installation with more than one user and asks first;
|
|
||||||
`backup:user:restore --mode=replace` asks before deleting an existing account (`--force` for
|
|
||||||
scripts in both cases).
|
|
||||||
- A failed scheduled backup is retried at most three times a day, an hour apart.
|
|
||||||
- The recovery kit carries the deletion delay; the kit file is created owner-only.
|
|
||||||
- Adding a location through the web API needs password confirmation.
|
|
||||||
|
|
||||||
### Fixed
|
|
||||||
- Exports of users whose id contains `.part-` could be deleted by cleanup on remote locations.
|
|
||||||
- Cleanup no longer frees the data of a per-user export whose manifest is missing.
|
|
||||||
- Crash safety of moving snapshots to and from the trash, of catalog writes by concurrent
|
|
||||||
backups and per-user exports, and of short uploads.
|
|
||||||
- A file that keeps changing no longer makes a backup restart forever.
|
|
||||||
- Long restores and exports keep their lock on the location alive.
|
|
||||||
- Floating-point database columns are dumped exactly.
|
|
||||||
|
|
||||||
### Known limitations
|
|
||||||
See "Known limitations" in the README: memory use on very large instances, never two servers
|
|
||||||
writing to one location, removing a passphrase slot does not revoke access, no rollback detection
|
|
||||||
before a disaster recovery, and a few smaller points. None of them block the road to 1.0.0; they
|
|
||||||
are on the backlog in `PLAN.md`.
|
|
||||||
|
|
||||||
## [0.1.0-alpha.1] - 2026-10-05
|
|
||||||
|
|
||||||
First public alpha. Feature-complete for phase 1: scheduled, encrypted, incremental backups, and
|
|
||||||
restore of files and folders. **Not for production use** — test on a copy of production (dev,
|
|
||||||
test, staging). Full disaster recovery (`backup:restore:full` on a fresh installation) is phase 2
|
|
||||||
and not implemented yet.
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- Encrypted (XChaCha20-Poly1305), content-addressed, deduplicated, incremental backups, streamed
|
|
||||||
from source to destination without a local copy of the data.
|
|
||||||
- Backup locations via files_external's own backend/auth classes, without ever mounting the
|
|
||||||
location: S3 (and compatible services), SFTP, WebDAV, FTP, Swift, or a local path.
|
|
||||||
- A recovery kit with up to three passphrase slots, mandatory download-and-confirm before the
|
|
||||||
first backup runs, and an optional encrypted copy by email.
|
|
||||||
- Resumable, time-boxed background jobs (database dump, file walk, upload), so a run survives
|
|
||||||
short PHP time limits and a crash is resumed, never restarted, without leaving the instance in
|
|
||||||
maintenance mode.
|
|
||||||
- Database dump and restore without external tools, via Nextcloud's own `IDBConnection`/DBAL,
|
|
||||||
streamed through the same pipeline as file data.
|
|
||||||
- Restore of files and folders via the web interface and `occ`, into a new folder, merged, or
|
|
||||||
replacing the original (with the previous content going to the trash).
|
|
||||||
- Retention policy (last/daily/weekly/monthly), prune with pack repacking, and a snapshot trash
|
|
||||||
with a configurable deletion delay.
|
|
||||||
- Append-only locations and documented recommendations against ransomware and a compromised
|
|
||||||
server (object lock, delete-less credentials, off-site copies).
|
|
||||||
- Admin settings page (locations, schedule, retention, key management) and a JSON API.
|
|
||||||
- Notifications and an admin setup check for a failed or overdue backup.
|
|
||||||
- CI: unit tests on PHP 8.2–8.4, and an end-to-end backup→restore round trip on Nextcloud 32–35 ×
|
|
||||||
SQLite/MariaDB/PostgreSQL, and against real S3 and SFTP backends (not just a local path).
|
|
||||||
|
|
||||||
### Known limitations
|
|
||||||
- No full/disaster-recovery restore yet (phase 2).
|
|
||||||
- No per-user backup/restore via user_migration yet (phase 2).
|
|
||||||
- No FTP/SMB coverage in CI yet, and no verification/checksum-audit command yet (phase 2).
|
|
||||||
- See the "Storage backend experiences" section of the README for provider-specific quirks found
|
|
||||||
so far (e.g. Hetzner Storage Box sub-accounts).
|
|
||||||
|
|
||||||
[0.9.6-beta.7]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.6-beta.7
|
|
||||||
[0.9.5-beta.6]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.5-beta.6
|
|
||||||
[0.9.4-beta.5]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.4-beta.5
|
|
||||||
[0.9.3-beta.4]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.3-beta.4
|
|
||||||
[0.9.2-beta.3]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.2-beta.3
|
|
||||||
[0.9.1-beta.2]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.1-beta.2
|
|
||||||
[0.9.0-beta.1]: https://github.com/dillardblom/ng_backup/releases/tag/v0.9.0-beta.1
|
|
||||||
[0.1.0-alpha.1]: https://github.com/dillardblom/ng_backup/releases/tag/v0.1.0-alpha.1
|
|
||||||
@@ -214,8 +214,8 @@ Daarom:
|
|||||||
### 3.8 Backup en restore per gebruiker: via user_migration
|
### 3.8 Backup en restore per gebruiker: via user_migration
|
||||||
|
|
||||||
In plaats van zelf te bepalen welke gegevens van een gebruiker meegaan, hergebruiken we
|
In plaats van zelf te bepalen welke gegevens van een gebruiker meegaan, hergebruiken we
|
||||||
**user_migration** (officiële Nextcloud-app, actief onderhouden, NC29–36; draait ook op onze eigen productie-instantie).
|
**user_migration** (officiële Nextcloud-app, actief onderhouden, NC29–36; staat ook op nc01).
|
||||||
Die app werkt met *migrators* die andere apps zelf aanleveren. Op onze productie-instantie zijn dat er al:
|
Die app werkt met *migrators* die andere apps zelf aanleveren. Op nc01 hebben die er al:
|
||||||
calendar, dav (agenda's en contactpersonen), deck, files_trashbin, mail, settings (profiel,
|
calendar, dav (agenda's en contactpersonen), deck, files_trashbin, mail, settings (profiel,
|
||||||
account, app-instellingen) en de bestanden zelf. Elke app die later een migrator toevoegt, gaat
|
account, app-instellingen) en de bestanden zelf. Elke app die later een migrator toevoegt, gaat
|
||||||
automatisch mee.
|
automatisch mee.
|
||||||
@@ -276,7 +276,7 @@ onderdelen wel en niet terugkomen, op basis van de geregistreerde migrators.
|
|||||||
| `backup:list [--target=]` | herstelpunten |
|
| `backup:list [--target=]` | herstelpunten |
|
||||||
| `backup:verify <snapshot> [--deep]` | checksums controleren, optioneel alles ontsleutelen |
|
| `backup:verify <snapshot> [--deep]` | checksums controleren, optioneel alles ontsleutelen |
|
||||||
| `backup:restore:file <snapshot> <user> <pad> [--to=]` | bestand/map terugzetten |
|
| `backup:restore:file <snapshot> <user> <pad> [--to=]` | bestand/map terugzetten |
|
||||||
| `backup:user:restore <snapshot> <user> [--to-user=]` | gebruiker terugzetten via user_migration (fase 2) |
|
| `backup:restore:user <snapshot> <user> [--to-user=]` | gebruiker terugzetten via user_migration (fase 2) |
|
||||||
| `backup:restore:full <snapshot>` | volledige restore (fase 2, verse installatie) |
|
| `backup:restore:full <snapshot>` | volledige restore (fase 2, verse installatie) |
|
||||||
| `backup:target:add/list/remove/test` | opslagdoelen beheren en testen |
|
| `backup:target:add/list/remove/test` | opslagdoelen beheren en testen |
|
||||||
| `backup:key:export / import` | herstelkit |
|
| `backup:key:export / import` | herstelkit |
|
||||||
@@ -308,14 +308,14 @@ niet met de hunne, maar bij installatie waarschuwen als `backup` ook actief is.
|
|||||||
| 2 | Backup/restore per gebruiker via user_migration, volledige restore op verse installatie, FTP/SMB, verificatie | 3–4 weken |
|
| 2 | Backup/restore per gebruiker via user_migration, volledige restore op verse installatie, FTP/SMB, verificatie | 3–4 weken |
|
||||||
| 3 | In-place volledige restore (restore-modus), zelfbediening voor gebruikers, extra doelen | open |
|
| 3 | In-place volledige restore (restore-modus), zelfbediening voor gebruikers, extra doelen | open |
|
||||||
|
|
||||||
Release in de app store pas na fase 1 en een periode van eigen gebruik op onze productie-instantie (net als occweb):
|
Release in de app store pas na fase 1 en een periode van eigen gebruik op nc01 (net als occweb):
|
||||||
een backup-app die niet betrouwbaar is, is erger dan geen backup-app.
|
een backup-app die niet betrouwbaar is, is erger dan geen backup-app.
|
||||||
|
|
||||||
## 8. Besluiten (2026-10-01)
|
## 8. Besluiten (2026-10-01)
|
||||||
|
|
||||||
1. **Doelgroep:** alle varianten: AIO, losse installaties én gehoste Nextcloud zonder SSH.
|
1. **Doelgroep:** alle varianten: AIO, losse installaties én gehoste Nextcloud zonder SSH.
|
||||||
2. **Licentie:** AGPL-3.0-or-later.
|
2. **Licentie:** AGPL-3.0-or-later.
|
||||||
3. **Repository:** eigen Gitea + GitHub (dillardblom), zoals occweb.
|
3. **Repository:** Ensembia op Gitea + GitHub (dillardblom), zoals occweb.
|
||||||
4. **Naam:** "NG Backup", app-id `ng_backup`, occ-voorvoegsel `backup:`.
|
4. **Naam:** "NG Backup", app-id `ng_backup`, occ-voorvoegsel `backup:`.
|
||||||
5. **Herstelkit:** verplicht downloaden + expliciete bevestiging door de beheerder (vinkje +
|
5. **Herstelkit:** verplicht downloaden + expliciete bevestiging door de beheerder (vinkje +
|
||||||
"Yes, I confirm"), zie 3.2; versleuteld per e-mail als optionele extra kopie.
|
"Yes, I confirm"), zie 3.2; versleuteld per e-mail als optionele extra kopie.
|
||||||
@@ -325,67 +325,6 @@ een backup-app die niet betrouwbaar is, is erger dan geen backup-app.
|
|||||||
export, verwijderen, opnieuw aanmaken), óf terugzetten als `<gebruiker>-bak`, zie 3.8.
|
export, verwijderen, opnieuw aanmaken), óf terugzetten als `<gebruiker>-bak`, zie 3.8.
|
||||||
8. **Fase 0 afgerond (2026-10-01),** resultaten in `spikes/RESULTS.md`.
|
8. **Fase 0 afgerond (2026-10-01),** resultaten in `spikes/RESULTS.md`.
|
||||||
|
|
||||||
## 9. Releases en versies (besluit 2026-10-02)
|
|
||||||
|
|
||||||
| Versie | Betekenis |
|
|
||||||
|---|---|
|
|
||||||
| 0.1.0-alpha.N | publiek; meetesten gewaardeerd op een kopie van productie (dev, test, staging) |
|
|
||||||
| 0.9.0-beta.N | mag meedraaien op productie, naast de reguliere backup; volledige restore aanwezig en getest |
|
|
||||||
| 1.0.0-rc.N | releasekandidaten, alleen bugfixes |
|
|
||||||
| 1.0.0 | stabiel: vervanging van de reguliere backup, noodherstel bewezen |
|
|
||||||
| 1.2.0 e.v. | functies van fase 2 (restore per gebruiker, FTP/SMB, verificatie, ...) |
|
|
||||||
|
|
||||||
Volgorde: fase 1 grondig testen, reviewen en debuggen; dan nadenken over 0.1.0-alpha.1. Een
|
|
||||||
minimale volledige restore (`occ backup:restore:full` op een verse installatie) komt vóór de beta.
|
|
||||||
|
|
||||||
**Roadmap-indeling fase 2 (besluit 2026-10-05):** naar v1.2.0 toewerken met in fase 2:
|
|
||||||
1. `backup:restore:full` (disaster recovery, verse installatie) — **klaar** (2026-10-05)
|
|
||||||
2. FTP/SMB-ondersteuning — **klaar** (2026-10-06): beide liepen al via files_external's generieke
|
|
||||||
adapter zonder enige ng_backup-code; alleen testdekking toegevoegd
|
|
||||||
(`tests/integration/targets-matrix.sh`) + bevindingen in README ("Storage backend experiences").
|
|
||||||
SMB vereist de `smbclient` PECL-extensie (niet standaard in de Nextcloud-image, in CI nu
|
|
||||||
on-the-fly gebouwd). NFS is geen eigen backend: lokaal mounten + `local`-backend erop richten.
|
|
||||||
3. Verificatie/checksum-audit-commando — **klaar** (2026-10-05)
|
|
||||||
4. Retentie/prune voor user-exports — **klaar** (2026-10-06): `occ backup:retention --user-last=N`
|
|
||||||
(standaard 3, per installatie, net als de bestaande snapshot-policy), toegepast door
|
|
||||||
`occ backup:prune` (geen prullenbak voor user-exports, in tegenstelling tot snapshots — on-demand,
|
|
||||||
niet het primaire vangnet).
|
|
||||||
5. Automatische user-export in het reguliere schema — **naar backlog** (niet voor deze release; on-demand blijft)
|
|
||||||
9. SMB-roundtrip (protocol, niet een specifieke server) — **klaar** (2026-10-06): CI-matrix, 2 GB-roundtrip en
|
|
||||||
user-export via SMB; NFS via de OMV-share getest (`local`-backend op een gemounte share, 2026-10-06)
|
|
||||||
10. `backup:user:restore` zonder user_migration: vraagt om toestemming om de app te installeren en te activeren (of
|
|
||||||
`--install-user-migration`) — **klaar** (2026-10-06); waarschuwing over de settings-allowlist — **klaar**;
|
|
||||||
agenda/contacten bij migratie naar een server zonder die gebruiker — **getest** (2026-10-06, naar `migrated-personal`/`migrated-contacts`)
|
|
||||||
Backlog: (a) de originele weergavenaam herstellen (het label uit de export is niet getest als
|
|
||||||
weergavenaam); (b) de interne `migrated-`-naam hernoemen (vereist directe databasewijziging, niet voor de beta).
|
|
||||||
|
|
||||||
**Backlog uit de pre-beta review (2026-10-07)** — geen belemmering voor 0.9.0-beta of 1.0.0, staat
|
|
||||||
in README "Known limitations":
|
|
||||||
1. Geheugen bij grote instanties (>1 TB, miljoenen bestanden): blob-index, prune-set, browse van een
|
|
||||||
snapshot in de web-UI, `trashExtras` bij restore en het manifest van een user-export staan volledig
|
|
||||||
in het geheugen; streaming/paginering nodig.
|
|
||||||
2. Twee servers op één locatie: catalogus kan breken (generatie overschreven, of anchor >200 generaties
|
|
||||||
achter). Docs-waarschuwing staat er; mogelijk later een echte put-if-absent per backend.
|
|
||||||
3. Sleutelrotatie: een slot verwijderen trekt geen toegang in (master-key blijft gelijk).
|
|
||||||
4. Rollback vóór DR niet detecteerbaar: referentie-generatie en -hash in de kit opnemen of tonen.
|
|
||||||
5. Kit-velden buiten `wrapped_key` (o.a. `delete_delay_days`) zijn niet geauthenticeerd; nu alleen getoond.
|
|
||||||
6. Repository-id in de AD van objecten (formaatwijziging).
|
|
||||||
7. Wachtwoorden van locaties via `--option-file` of omgevingsvariabele i.p.v. de commandoregel.
|
|
||||||
8. Tijdelijke restore-bestanden niet via de Files-API verwijderen (komen nu in de prullenbak).
|
|
||||||
9. DB-restore in foreign-key-volgorde (cascades).
|
|
||||||
10. Packs zonder index (index-upload mislukt) opruimen.
|
|
||||||
11. `backup:restore:full` in onderhoudsmodus en hervatbaar maken; nu: opnieuw draaien met `--force`.
|
|
||||||
12. Config/rename op FTP/SMB-servers die geen rename over een bestaand bestand toestaan.
|
|
||||||
|
|
||||||
Doorgeschoven naar **v1.3.0** (fase 3, niet meer fase 2): in-place volledige restore
|
|
||||||
(eigen restore-modus i.p.v. onderhoudsmodus) en zelfbediening voor gebruikers (eigen bestanden
|
|
||||||
terugzetten vanuit de Files-app).
|
|
||||||
|
|
||||||
**Let op (ontdekt 2026-10-05):** Nextcloud's eigen `info.xsd` staat voor `<version>` alleen
|
|
||||||
`[0-9]+(\.[0-9]+){0,2}` toe — geen `-alpha`/`-beta`/`-rc`-achtervoegsel. De stabiliteitsaanduiding
|
|
||||||
uit deze tabel gaat dus niet in `info.xml` (dat blijft gewoon oplopend, bijv. `0.1.0`), maar in de
|
|
||||||
git-tag (`v0.1.0-alpha.1`), de GitHub/Gitea release (gemarkeerd als pre-release) en de CHANGELOG.
|
|
||||||
|
|
||||||
## Bijlage: lessen uit nextcloud/backup (test NC34, 2026-10-01)
|
## Bijlage: lessen uit nextcloud/backup (test NC34, 2026-10-01)
|
||||||
|
|
||||||
- Externe programma's (`pg_dump`) en extensies (`pgsql`, `mysqli`) → onbruikbaar op standaard/hosted.
|
- Externe programma's (`pg_dump`) en extensies (`pgsql`, `mysqli`) → onbruikbaar op standaard/hosted.
|
||||||
|
|||||||
@@ -1,204 +1,8 @@
|
|||||||
# NG Backup
|
# NG Backup
|
||||||
|
|
||||||
Encrypted, incremental backups of Nextcloud (database, configuration and data) with restore
|
Encrypted, incremental backups of Nextcloud with restore from the web interface.
|
||||||
from the web interface. Works without shell access: no external programs, no extra PHP
|
Works without shell access: no external programs, no extra PHP extensions beyond sodium.
|
||||||
extensions beyond sodium for ng_backup itself. Runs on Nextcloud All-in-One, regular
|
|
||||||
installations and hosted Nextcloud. (Choosing SMB as a storage backend is the one exception:
|
|
||||||
see "Storage backend experiences" below — that requirement comes from files_external, not from
|
|
||||||
ng_backup, and applies to any Nextcloud app that writes to an SMB share.)
|
|
||||||
|
|
||||||
**Status: beta (0.9.6).** Tested thoroughly, usable alongside your current backup, but not yet a
|
**Status: phase 0 (prototype). Not usable yet.** See `PLAN.md` for the design.
|
||||||
drop-in replacement for it. See "Known limitations" below, `PLAN.md` for the design and
|
|
||||||
`docs/HOWTO.md` for a step-by-step guide from setup to a tested restore.
|
|
||||||
|
|
||||||
License: AGPL-3.0-or-later.
|
License: AGPL-3.0-or-later.
|
||||||
|
|
||||||
## How it works
|
|
||||||
|
|
||||||
- Backups go to a **location**: S3 (and S3-compatible storage), SFTP, WebDAV, FTP, Swift or a
|
|
||||||
local path. NG Backup uses the connection code of Nextcloud's External storage app, but never
|
|
||||||
mounts the location: it is only connected while a backup or restore runs, and it never
|
|
||||||
appears in Files, for anyone.
|
|
||||||
- Data is cut into blocks, deduplicated, compressed and encrypted (XChaCha20-Poly1305) before it
|
|
||||||
leaves the server, and streamed straight to the location: no local copy, no double disk space.
|
|
||||||
Unchanged files are not read again; only new data is uploaded.
|
|
||||||
- The database is dumped through Nextcloud's own connection, inside one consistent transaction.
|
|
||||||
- Restores go through Nextcloud, so overwritten files keep their old content as a version and
|
|
||||||
removed files go to the trash.
|
|
||||||
|
|
||||||
## Quick start
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:key:init --label="Safe" # first passphrase, deletion delay (default 7 days)
|
|
||||||
occ backup:key:slot:add --label="CTO" # optional second and third passphrase
|
|
||||||
occ backup:key:slot:add --label="Head of IT"
|
|
||||||
occ backup:key:kit --output=/var/tmp/ng_backup-kit.json
|
|
||||||
occ backup:key:confirm
|
|
||||||
occ backup:target:add offsite amazons3 -a amazons3::accesskey -o bucket=... -o key=... -o secret=...
|
|
||||||
occ backup:run
|
|
||||||
```
|
|
||||||
|
|
||||||
Or use **Administration settings → NG Backup**.
|
|
||||||
|
|
||||||
## The backup key, passphrases and the recovery kit
|
|
||||||
|
|
||||||
All backups of an installation are encrypted with one master key.
|
|
||||||
|
|
||||||
- The **server** keeps the master key, encrypted with the instance secret, so scheduled backups
|
|
||||||
and restores work without anyone typing a passphrase. Any administrator can restore from the
|
|
||||||
web interface. Consequence: whoever fully controls the server can read the backups; whoever
|
|
||||||
only gets hold of the backup location cannot.
|
|
||||||
- For a **restore on a new server** (disaster recovery) you need the **recovery kit** and **one
|
|
||||||
passphrase**. The kit contains the master key, encrypted separately with each passphrase.
|
|
||||||
- Up to **three passphrases** ("slots") can open the key, so one person leaving or dying does not
|
|
||||||
lock the company out. A suggested setup:
|
|
||||||
|
|
||||||
| Slot | Kept by |
|
|
||||||
|---|---|
|
|
||||||
| 1 | in the safe (sealed envelope) |
|
|
||||||
| 2 | the CTO |
|
|
||||||
| 3 | the head of IT |
|
|
||||||
|
|
||||||
Every holder keeps a copy of the same recovery kit and only knows their own passphrase. Using
|
|
||||||
a single slot is possible, but then that one passphrase is a single point of failure.
|
|
||||||
- When a holder leaves, give their slot a new passphrase (`occ backup:key:slot:replace`), download
|
|
||||||
the new kit and confirm it. Note: an old kit together with the old passphrase still opens the
|
|
||||||
same master key; to revoke it completely, start new locations with a new key. This is noted on
|
|
||||||
the backlog and will be addressed in a future version (key rotation).
|
|
||||||
- Downloading the kit, and every change to the slots, requires the administrator's password, is
|
|
||||||
written to the audit log and is announced to all administrators.
|
|
||||||
|
|
||||||
## Protection against ransomware and a compromised server
|
|
||||||
|
|
||||||
A backup is only useful if an attacker cannot destroy it. An attacker with administrator access
|
|
||||||
to Nextcloud could try to remove old restore points so that only infected data remains.
|
|
||||||
NG Backup makes this hard and visible, but **software alone can never guarantee it**: the
|
|
||||||
location must refuse deletions itself.
|
|
||||||
|
|
||||||
What NG Backup does:
|
|
||||||
|
|
||||||
- **Deletion delay.** Removed restore points first go to a trash on the location and are only
|
|
||||||
deleted after the delay chosen at installation (default 7 days, cannot be changed afterwards).
|
|
||||||
Within that time any administrator can bring them back (`occ backup:trash:restore`, or in the
|
|
||||||
web interface).
|
|
||||||
- **Notifications to all administrators** when restore points are removed, the retention is
|
|
||||||
changed, append-only is turned off, a location is removed or the recovery kit is downloaded,
|
|
||||||
and an entry in the audit log (admin_audit).
|
|
||||||
- **Append-only locations** (`occ backup:target:append-only NAME on`): NG Backup never deletes
|
|
||||||
anything there; old data is cleaned up by the location's own lifecycle rules.
|
|
||||||
- Removing a location in NG Backup only removes the connection. The backups on the location are
|
|
||||||
not touched and nothing keeps cleaning up in the background.
|
|
||||||
|
|
||||||
What you should do on the location (strongly recommended):
|
|
||||||
|
|
||||||
- **S3: enable Object Lock** on the bucket, in **compliance mode**, with a default retention of
|
|
||||||
for example **90 days**, plus versioning. Then nobody, not even with the server's credentials,
|
|
||||||
can delete or overwrite backups younger than 90 days. Combine with an append-only location in
|
|
||||||
NG Backup and a lifecycle rule that removes old versions after the lock expires.
|
|
||||||
With Object Lock, a deletion by NG Backup (or by an attacker through it) only adds a *delete
|
|
||||||
marker*: the locked data stays and can be brought back from the bucket's object versions until
|
|
||||||
the lock expires. Tested with an S3-compatible server; still to be verified on AWS itself, which
|
|
||||||
requires a checksum on uploads to locked buckets.
|
|
||||||
- Use **credentials without delete rights** for the backup location where the storage supports it.
|
|
||||||
- **SFTP / storage boxes:** enable the provider's own snapshots (e.g. daily, kept 30-90 days) and
|
|
||||||
give the backup user access to its own folder only, without a shell.
|
|
||||||
- Keep at least one location **off-site**, on a different network than the Nextcloud server.
|
|
||||||
|
|
||||||
## Restore
|
|
||||||
|
|
||||||
- **Files and folders** of a user: in the web interface or `occ backup:restore`, into a new folder
|
|
||||||
("Restored <date>"), merged into the original place, or replacing it.
|
|
||||||
- **Raw restore** of any path (for example `config/`) to a local directory: `occ backup:restore ... --to-directory=DIR`.
|
|
||||||
- **Whole server** (disaster recovery onto a fresh installation): `occ backup:restore:full`, see the how-to.
|
|
||||||
- **One user** (account, settings, files, calendars, contacts via user_migration): `occ backup:user:backup` and
|
|
||||||
`occ backup:user:restore`, replacing the account or restoring it as `<uid>-bak`. See the how-to.
|
|
||||||
|
|
||||||
## Storage backend experiences
|
|
||||||
|
|
||||||
Notes from actually running a backend, beyond what's in files_external's own documentation.
|
|
||||||
Contributions welcome via PR — add what you ran into with your own provider.
|
|
||||||
|
|
||||||
- **Hetzner Storage Box, sub-account:** the SFTP backend's `root` option must be `home`, not `/`.
|
|
||||||
A sub-account's SFTP login lands in `/home` (confirmed via `pwd()` after login); writing to the
|
|
||||||
literal chroot root (`/`) fails with a generic `NET_SFTP_STATUS_FAILURE` from the SFTP server,
|
|
||||||
while the same write under `home` (or a path below it) succeeds. This affects any backend option
|
|
||||||
form that lets you set a path relative to the account's root, not just ng_backup. Not yet
|
|
||||||
verified whether this also applies to a Hetzner Storage Box **main** account (only tested with a
|
|
||||||
sub-account so far) — if you've tried a main account, please say so in a PR.
|
|
||||||
|
|
||||||
- **Hetzner Storage Box over WebDAV and SMB (sub-accounts, 2026-10-07):** both work end to end
|
|
||||||
(backup, incremental run, restore of a folder, verify), tested from outside the Hetzner network
|
|
||||||
with about 66 MiB of data. In the Hetzner Console, enable WebDAV or SMB support for the
|
|
||||||
sub-account, and "External reachability" when your server is not in the Hetzner network.
|
|
||||||
WebDAV: host `https://uXXXXX-subN.your-storagebox.de`, and the `root`
|
|
||||||
must be a folder that already exists (or empty): WebDAV does not create missing parent folders,
|
|
||||||
so a root that doesn't exist yet fails `backup:target:add` with `Sabre\HTTP\ClientHttpException:
|
|
||||||
Conflict`. SMB: share = the sub-account name, port 445 must be reachable from your server (it was
|
|
||||||
from a home connection here, but some providers block it), and the `smbclient` PECL extension is
|
|
||||||
required (see the SMB entry below). Both have a noticeable fixed cost per run: an incremental run
|
|
||||||
that uploads nothing took about 20 s over WebDAV and 50 s over SMB. Over WebDAV, Nextcloud gives
|
|
||||||
every request at most 30 s by default, also an upload of a whole 32 MiB pack: on an upload
|
|
||||||
slower than about 9 Mbit/s a backup fails with `cURL error 28: Operation timed out`. Raise the
|
|
||||||
limit with `occ config:system:set davstorage.request_timeout --value=600 --type=integer`.
|
|
||||||
|
|
||||||
- **FTP, server-dependent absolute-path handling:** files_external's FTP backend always builds
|
|
||||||
absolute paths (e.g. `MKD /repo`) for the configured root, never relative ones. `atmoz/sftp`-style
|
|
||||||
`vsftpd` test servers (chrooted) can reject that with a generic "create directory operation
|
|
||||||
failed", even though a plain `ftp_mkdir()` with a *relative* path on the same server succeeds;
|
|
||||||
`pure-ftpd` accepts the absolute form fine. If a working FTP server rejects ng_backup's writes,
|
|
||||||
try a different FTP server implementation before assuming the backend itself is broken.
|
|
||||||
|
|
||||||
- **SMB/CIFS *lists* as available with just the `smbclient` CLI binary, but does not actually
|
|
||||||
work for ng_backup without the `smbclient` PECL extension too.** files_external falls back to a
|
|
||||||
shell-wrapped `smbclient` process when the native extension is missing, which registers the
|
|
||||||
backend in `occ backup:target:backends` fine, but that fallback's stream cannot seek. ng_backup
|
|
||||||
reads blobs back by byte range while backing up, so even the very first (small) backup fails
|
|
||||||
with `Seek failed in packs/...`. Install the extension (`libsmbclient-dev` + `pecl install
|
|
||||||
smbclient` + `docker-php-ext-enable smbclient` on Debian-based images), not just the CLI tool —
|
|
||||||
the backend being listed is not proof it will work here. Once the extension is present it works
|
|
||||||
the same as any other backend; no ng_backup-specific config beyond host/share/user/password.
|
|
||||||
|
|
||||||
- **NFS is not a files_external backend at all.** Mount the NFS export on the server yourself
|
|
||||||
(e.g. via `/etc/fstab` or a systemd `.mount` unit) and point ng_backup's `local` backend at the
|
|
||||||
mount point; ng_backup never mounts or unmounts anything on its own, so an NFS mount dropping
|
|
||||||
mid-run behaves like any other local-path failure (the run resumes). See PLAN.md 3.5 for why NFS
|
|
||||||
can't be driven from PHP directly.
|
|
||||||
|
|
||||||
## Known limitations
|
|
||||||
|
|
||||||
On the backlog; none of these block the beta or 1.0.0, but know them before you rely on it:
|
|
||||||
|
|
||||||
- **Very large instances** (well over 1 TB, millions of files): the blob index, cleanup, browsing a
|
|
||||||
snapshot in the web interface and per-user export manifests are held in memory. Expect high
|
|
||||||
memory use there; reports from such instances are very welcome.
|
|
||||||
- **One location, one server.** Never let two Nextcloud servers write to the same location (for
|
|
||||||
example a disaster recovery test server pointed at the production location): they can break each
|
|
||||||
other's catalog, and the location is then refused as possibly tampered with. Test a disaster
|
|
||||||
recovery against a copy of the location, or only read from it.
|
|
||||||
- **Removing a passphrase slot does not revoke access.** The master key stays the same, so someone
|
|
||||||
with an old recovery kit and that slot's passphrase can still decrypt the location if they can
|
|
||||||
read it. Key rotation is not available yet.
|
|
||||||
- **Rollback before a disaster recovery is not detected.** A fresh server has no record of the
|
|
||||||
newest catalog generation, so a location rolled back to an older state looks valid there.
|
|
||||||
- Location passwords passed with `-o password=...` are visible in the process list and shell
|
|
||||||
history while the command runs.
|
|
||||||
- Temporary files of a restore into a user's files end up in that user's trash.
|
|
||||||
- Restoring the database does not order tables by foreign keys; apps with cascading foreign keys
|
|
||||||
could lose rows (Nextcloud core does not use them).
|
|
||||||
- A pack whose index upload failed stays on the location until removed by hand.
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
| Command | Purpose |
|
|
||||||
|---|---|
|
|
||||||
| `backup:key:init`, `backup:key:kit`, `backup:key:confirm` | create the key, recovery kit, confirmation |
|
|
||||||
| `backup:key:slots`, `backup:key:slot:add/replace/remove` | passphrase slots |
|
|
||||||
| `backup:target:backends/add/edit/list/remove/append-only` | backup locations |
|
|
||||||
| `backup:run`, `backup:status`, `backup:list` | back up, status, restore points |
|
|
||||||
| `backup:cancel <run>` | stop a running backup |
|
|
||||||
| `backup:browse`, `backup:restore` | browse and restore |
|
|
||||||
| `backup:retention`, `backup:prune`, `backup:trash`, `backup:trash:restore` | retention, cleanup, trash |
|
|
||||||
| `backup:verify` | checksum audit of a restore point (`--deep` downloads and decrypts everything) |
|
|
||||||
| `backup:restore:full` | whole-server restore onto a fresh installation (after `backup:key:import`) |
|
|
||||||
| `backup:key:import` | import a recovery kit on a fresh installation |
|
|
||||||
| `backup:user:backup`, `backup:user:restore` | one user via user_migration (asks before installing it) |
|
|
||||||
|
|||||||
+3
-33
@@ -4,25 +4,13 @@
|
|||||||
<id>ng_backup</id>
|
<id>ng_backup</id>
|
||||||
<name>NG Backup</name>
|
<name>NG Backup</name>
|
||||||
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
<summary>Encrypted, incremental backups of your Nextcloud with restore from the web interface</summary>
|
||||||
<description><![CDATA[Beta: scheduled, encrypted, incremental backups of the database, config and data
|
<description><![CDATA[Work in progress, not ready for use.]]></description>
|
||||||
directory to S3, SFTP, WebDAV, FTP, SMB or a local path, with restore of files and folders, of
|
<version>0.0.5</version>
|
||||||
single users (via user_migration) and of the whole server onto a fresh installation. Tested
|
|
||||||
thoroughly and usable alongside your current backup, but not yet a drop-in replacement for it.
|
|
||||||
See the README for known limitations and the recovery kit before relying on it.]]></description>
|
|
||||||
<version>0.9.6</version>
|
|
||||||
<licence>agpl</licence>
|
<licence>agpl</licence>
|
||||||
<author mail="dillard.blom@ensembia.com">Dillard Blom</author>
|
<author>Dillard Blom</author>
|
||||||
<namespace>NgBackup</namespace>
|
<namespace>NgBackup</namespace>
|
||||||
<documentation>
|
|
||||||
<admin>https://github.com/dillardblom/ng_backup/blob/main/docs/HOWTO.md</admin>
|
|
||||||
</documentation>
|
|
||||||
<category>tools</category>
|
<category>tools</category>
|
||||||
<website>https://github.com/dillardblom/ng_backup</website>
|
|
||||||
<bugs>https://github.com/dillardblom/ng_backup/issues</bugs>
|
<bugs>https://github.com/dillardblom/ng_backup/issues</bugs>
|
||||||
<repository type="git">https://github.com/dillardblom/ng_backup</repository>
|
|
||||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/1-backups.png</screenshot>
|
|
||||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/2-restore.png</screenshot>
|
|
||||||
<screenshot>https://raw.githubusercontent.com/dillardblom/ng_backup/main/screenshots/3-settings.png</screenshot>
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
<php min-version="8.1"/>
|
<php min-version="8.1"/>
|
||||||
<lib>sodium</lib>
|
<lib>sodium</lib>
|
||||||
@@ -37,31 +25,13 @@ See the README for known limitations and the recovery kit before relying on it.]
|
|||||||
<command>OCA\NgBackup\Command\KeyConfirm</command>
|
<command>OCA\NgBackup\Command\KeyConfirm</command>
|
||||||
<command>OCA\NgBackup\Command\TargetBackends</command>
|
<command>OCA\NgBackup\Command\TargetBackends</command>
|
||||||
<command>OCA\NgBackup\Command\TargetAdd</command>
|
<command>OCA\NgBackup\Command\TargetAdd</command>
|
||||||
<command>OCA\NgBackup\Command\TargetEdit</command>
|
|
||||||
<command>OCA\NgBackup\Command\TargetList</command>
|
<command>OCA\NgBackup\Command\TargetList</command>
|
||||||
<command>OCA\NgBackup\Command\TargetRemove</command>
|
<command>OCA\NgBackup\Command\TargetRemove</command>
|
||||||
<command>OCA\NgBackup\Command\RunBackup</command>
|
<command>OCA\NgBackup\Command\RunBackup</command>
|
||||||
<command>OCA\NgBackup\Command\CancelRun</command>
|
|
||||||
<command>OCA\NgBackup\Command\Status</command>
|
<command>OCA\NgBackup\Command\Status</command>
|
||||||
<command>OCA\NgBackup\Command\ListSnapshots</command>
|
<command>OCA\NgBackup\Command\ListSnapshots</command>
|
||||||
<command>OCA\NgBackup\Command\Browse</command>
|
<command>OCA\NgBackup\Command\Browse</command>
|
||||||
<command>OCA\NgBackup\Command\Restore</command>
|
<command>OCA\NgBackup\Command\Restore</command>
|
||||||
<command>OCA\NgBackup\Command\Prune</command>
|
|
||||||
<command>OCA\NgBackup\Command\Retention</command>
|
|
||||||
<command>OCA\NgBackup\Command\TargetAppendOnly</command>
|
|
||||||
<command>OCA\NgBackup\Command\TargetTrustCurrent</command>
|
|
||||||
<command>OCA\NgBackup\Command\TargetLimit</command>
|
|
||||||
<command>OCA\NgBackup\Command\Trash</command>
|
|
||||||
<command>OCA\NgBackup\Command\TrashRestore</command>
|
|
||||||
<command>OCA\NgBackup\Command\KeySlots</command>
|
|
||||||
<command>OCA\NgBackup\Command\KeySlotAdd</command>
|
|
||||||
<command>OCA\NgBackup\Command\KeySlotReplace</command>
|
|
||||||
<command>OCA\NgBackup\Command\KeySlotRemove</command>
|
|
||||||
<command>OCA\NgBackup\Command\BackupUser</command>
|
|
||||||
<command>OCA\NgBackup\Command\RestoreUser</command>
|
|
||||||
<command>OCA\NgBackup\Command\KeyImport</command>
|
|
||||||
<command>OCA\NgBackup\Command\RestoreFull</command>
|
|
||||||
<command>OCA\NgBackup\Command\Verify</command>
|
|
||||||
</commands>
|
</commands>
|
||||||
<settings>
|
<settings>
|
||||||
<admin>OCA\NgBackup\Settings\Admin</admin>
|
<admin>OCA\NgBackup\Settings\Admin</admin>
|
||||||
|
|||||||
@@ -11,18 +11,10 @@ return [
|
|||||||
['name' => 'AdminApi#initKey', 'url' => '/api/key', 'verb' => 'POST'],
|
['name' => 'AdminApi#initKey', 'url' => '/api/key', 'verb' => 'POST'],
|
||||||
['name' => 'AdminApi#downloadKit', 'url' => '/api/key/kit', 'verb' => 'GET'],
|
['name' => 'AdminApi#downloadKit', 'url' => '/api/key/kit', 'verb' => 'GET'],
|
||||||
['name' => 'AdminApi#confirmKit', 'url' => '/api/key/confirm', 'verb' => 'POST'],
|
['name' => 'AdminApi#confirmKit', 'url' => '/api/key/confirm', 'verb' => 'POST'],
|
||||||
['name' => 'AdminApi#addSlot', 'url' => '/api/key/slots', 'verb' => 'POST'],
|
|
||||||
['name' => 'AdminApi#replaceSlot', 'url' => '/api/key/slots/{slot}', 'verb' => 'PUT'],
|
|
||||||
['name' => 'AdminApi#removeSlot', 'url' => '/api/key/slots/{slot}', 'verb' => 'DELETE'],
|
|
||||||
['name' => 'AdminApi#trash', 'url' => '/api/targets/{targetId}/trash', 'verb' => 'GET'],
|
|
||||||
['name' => 'AdminApi#untrash', 'url' => '/api/targets/{targetId}/trash/{snapshotId}', 'verb' => 'POST'],
|
|
||||||
['name' => 'AdminApi#backends', 'url' => '/api/backends', 'verb' => 'GET'],
|
['name' => 'AdminApi#backends', 'url' => '/api/backends', 'verb' => 'GET'],
|
||||||
['name' => 'AdminApi#addTarget', 'url' => '/api/targets', 'verb' => 'POST'],
|
['name' => 'AdminApi#addTarget', 'url' => '/api/targets', 'verb' => 'POST'],
|
||||||
['name' => 'AdminApi#targetSettings', 'url' => '/api/targets/{id}/settings', 'verb' => 'GET'],
|
|
||||||
['name' => 'AdminApi#updateTarget', 'url' => '/api/targets/{id}', 'verb' => 'PUT'],
|
|
||||||
['name' => 'AdminApi#removeTarget', 'url' => '/api/targets/{id}', 'verb' => 'DELETE'],
|
['name' => 'AdminApi#removeTarget', 'url' => '/api/targets/{id}', 'verb' => 'DELETE'],
|
||||||
['name' => 'AdminApi#testTarget', 'url' => '/api/targets/{id}/test', 'verb' => 'POST'],
|
['name' => 'AdminApi#testTarget', 'url' => '/api/targets/{id}/test', 'verb' => 'POST'],
|
||||||
['name' => 'AdminApi#setTargetLimit', 'url' => '/api/targets/{id}/limit', 'verb' => 'PUT'],
|
|
||||||
['name' => 'AdminApi#startBackup', 'url' => '/api/backups', 'verb' => 'POST'],
|
['name' => 'AdminApi#startBackup', 'url' => '/api/backups', 'verb' => 'POST'],
|
||||||
['name' => 'AdminApi#setSchedule', 'url' => '/api/schedule', 'verb' => 'PUT'],
|
['name' => 'AdminApi#setSchedule', 'url' => '/api/schedule', 'verb' => 'PUT'],
|
||||||
['name' => 'AdminApi#snapshots', 'url' => '/api/targets/{targetId}/snapshots', 'verb' => 'GET'],
|
['name' => 'AdminApi#snapshots', 'url' => '/api/targets/{targetId}/snapshots', 'verb' => 'GET'],
|
||||||
|
|||||||
-174
@@ -1,174 +0,0 @@
|
|||||||
# NG Backup how-to
|
|
||||||
|
|
||||||
Step by step: from first install to a tested restore. All commands run as the web user
|
|
||||||
(`sudo -u www-data php occ ...`, or `docker exec -u www-data <container> php occ ...` in
|
|
||||||
All-in-One and Docker setups). Replace names in `<angle brackets>` with your own.
|
|
||||||
|
|
||||||
## 1. Create the key and the recovery kit
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:key:init --label="Main" --delete-delay-days=7
|
|
||||||
occ backup:key:kit --output=/var/tmp/ng_backup-kit.json
|
|
||||||
occ backup:key:confirm -u <your-admin-uid> --code=<code shown> --confirm="Yes, I confirm"
|
|
||||||
```
|
|
||||||
|
|
||||||
- The passphrase protects the backup. Keep it somewhere other than this server.
|
|
||||||
- Optional: extra passphrases for other people with `occ backup:key:slot:add --label="..."`.
|
|
||||||
Any one passphrase is enough to restore. Removing a slot later does not revoke access for someone
|
|
||||||
who kept an old kit and that passphrase (see "Known limitations" in the README).
|
|
||||||
- The recovery kit (`/var/tmp/ng_backup-kit.json` above) is also required for disaster recovery. The web
|
|
||||||
user writes it there; then move it off the server as root, next to the passphrase, and delete the
|
|
||||||
copy in `/var/tmp`. Without the kit and a passphrase, the backup cannot be read.
|
|
||||||
|
|
||||||
## 2. Add a backup location
|
|
||||||
|
|
||||||
NG Backup uses Nextcloud's External storage backends. Check what your server supports first:
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:target:backends
|
|
||||||
```
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:target:add offsite amazons3 -a amazons3::accesskey -o bucket=<bucket> -o key=<key> -o secret=<secret>
|
|
||||||
occ backup:target:add nas sftp -a password::password -o host=<host> -o user=<user> -o password=<pw> -o root=home
|
|
||||||
occ backup:target:add local local -a null::null -o datadir=/mnt/backup
|
|
||||||
```
|
|
||||||
|
|
||||||
- **SFTP on a Hetzner Storage Box sub-account:** use `-o root=home`, not `/` (see README).
|
|
||||||
- **FTP:** use a server that accepts absolute paths (for example `pure-ftpd`).
|
|
||||||
- **SMB:** requires the native `smbclient` PHP extension. The CLI binary alone is not enough (see README).
|
|
||||||
`backup:target:add` refuses SMB when the extension is missing.
|
|
||||||
- **NFS:** not a backend of its own. Mount the export on the server and use the `local` backend
|
|
||||||
with the mount point as `datadir`.
|
|
||||||
|
|
||||||
Verify the location before relying on it:
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:target:list
|
|
||||||
occ backup:status
|
|
||||||
```
|
|
||||||
|
|
||||||
A backup that hangs or takes far too long can be stopped. `backup:status` shows the run number;
|
|
||||||
the command waits until the step in progress is finished, then the process working on the run
|
|
||||||
stops, and the next cleanup removes what it stored:
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:cancel 7
|
|
||||||
```
|
|
||||||
|
|
||||||
Changed password, new host name, or moved the repository? Edit the location instead of adding a
|
|
||||||
new one, so its history stays (in the web interface: **Edit** next to the location). The new
|
|
||||||
settings must lead to the same repository; options you do not pass keep their value:
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:target:edit nas -o password=<new pw>
|
|
||||||
```
|
|
||||||
|
|
||||||
## 3. Run the first backup
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:run offsite -l first
|
|
||||||
occ backup:list
|
|
||||||
```
|
|
||||||
|
|
||||||
Later runs upload only changed data. An unchanged run uploads (almost) nothing; the output shows
|
|
||||||
the number of uploaded bytes, so check that value when you test incremental behaviour.
|
|
||||||
|
|
||||||
Schedule `occ backup:run` from system cron (or use the Administration settings, if you prefer).
|
|
||||||
|
|
||||||
## 4. Check the backup regularly
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:verify offsite <snapshot> # shallow: all blocks are present
|
|
||||||
occ backup:verify offsite <snapshot> --deep # also downloads and decrypts every block
|
|
||||||
```
|
|
||||||
|
|
||||||
Run the deep check at least once after setting up a location, and after changing it.
|
|
||||||
|
|
||||||
## 5. Restore files and folders
|
|
||||||
|
|
||||||
In the web interface, or with occ:
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:browse offsite <snapshot>
|
|
||||||
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=new-folder
|
|
||||||
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=merge
|
|
||||||
occ backup:restore offsite <snapshot> data/<uid>/files/Docs --mode=replace
|
|
||||||
occ backup:restore offsite <snapshot> config --to-directory=/var/tmp/restore-config
|
|
||||||
```
|
|
||||||
|
|
||||||
- `new-folder` (default): puts the files in "Restored <date>" next to the original. Safest choice.
|
|
||||||
- `merge`: adds missing files, keeps current ones.
|
|
||||||
- `replace`: overwrites the folder. Replaced files keep their old content as a version, and removed
|
|
||||||
files go to the trash.
|
|
||||||
- `--to-directory`: writes raw files to a local directory (for example `config/`).
|
|
||||||
|
|
||||||
## 6. Back up and restore one user
|
|
||||||
|
|
||||||
Needs the `user_migration` app. NG Backup asks before installing it:
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:user:backup offsite <uid>
|
|
||||||
occ backup:user:restore offsite users/<uid>/<exportId> # replace (default)
|
|
||||||
occ backup:user:restore offsite users/<uid>/<exportId> --mode=as-backup
|
|
||||||
```
|
|
||||||
|
|
||||||
- If `user_migration` is missing, the command asks to install and enable it. For non-interactive
|
|
||||||
use, add `--install-user-migration` to approve that explicitly. The command stops after the
|
|
||||||
installation; run it once more.
|
|
||||||
- `replace` first makes a safety export of the current account, then recreates the user under the
|
|
||||||
same id. When the account exists, the command asks first; in scripts, add `--force`. Shares to
|
|
||||||
others and data of apps without a user_migration migrator are not restored, so prefer
|
|
||||||
`as-backup` when in doubt: it restores into `<uid>-bak` and leaves the original untouched.
|
|
||||||
- A restore onto a server where the user does not exist works. Calendars and address books get an
|
|
||||||
internal name with the prefix `migrated-` (for example `migrated-personal`). That prefix comes from
|
|
||||||
user_migration and cannot be turned off. The user can change the display name of these calendars and
|
|
||||||
address books in the app as usual; the internal name stays as it is.
|
|
||||||
- **Settings:** on restore, `user_migration` imports only the app settings on its own allowlist
|
|
||||||
(for example calendar view and reminder settings). Other app settings of the user are skipped.
|
|
||||||
NG Backup reuses `user_migration`'s importer, so this list is not extended.
|
|
||||||
- Keep the last few exports per user: `occ backup:retention --user-last=3`.
|
|
||||||
|
|
||||||
## 7. Disaster recovery: restore the whole server
|
|
||||||
|
|
||||||
Use a fresh Nextcloud installation of the same major version, with NG Backup installed and
|
|
||||||
the same location available. For a test, use a **copy** of the location: two servers writing to one
|
|
||||||
location break each other's catalog.
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:key:import /path/ng_backup-kit.json --passphrase-file=/path/passphrase
|
|
||||||
occ backup:target:add offsite <backend> ... # same details as on the original server
|
|
||||||
occ backup:list offsite # the snapshots must be listed
|
|
||||||
occ backup:restore:full offsite <snapshot>
|
|
||||||
```
|
|
||||||
|
|
||||||
The command refuses on an installation that already has more than one user, and asks before it
|
|
||||||
starts; `--force` skips both (also needed to rerun it after it stopped halfway).
|
|
||||||
|
|
||||||
Afterwards, check that users, files and the database are complete, and run a backup of the restored
|
|
||||||
server. The restored installation keeps its own app configuration and secret.
|
|
||||||
|
|
||||||
## 8. Retention
|
|
||||||
|
|
||||||
```
|
|
||||||
occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12
|
|
||||||
occ backup:prune offsite --dry-run # show first
|
|
||||||
occ backup:prune offsite # apply the policy; forgotten snapshots go to the trash first
|
|
||||||
occ backup:trash # list the trash
|
|
||||||
occ backup:trash:restore offsite <id>
|
|
||||||
```
|
|
||||||
|
|
||||||
Snapshots are kept according to the policy, and their data is deleted only after the deletion delay
|
|
||||||
(set with `backup:key:init --delete-delay-days`). `occ backup:prune offsite --dry-run` shows what would be
|
|
||||||
removed without changing anything.
|
|
||||||
|
|
||||||
## 9. Checklist before you rely on it
|
|
||||||
|
|
||||||
- [ ] Key created, passphrase stored off-server, recovery kit stored off-server
|
|
||||||
- [ ] Location added; `backup:status` shows it reachable
|
|
||||||
- [ ] First backup done and listed in `backup:list`
|
|
||||||
- [ ] `backup:verify --deep` succeeds
|
|
||||||
- [ ] One file restore tested (`new-folder`)
|
|
||||||
- [ ] Disaster recovery tested on a second, fresh installation (or in a test environment)
|
|
||||||
@@ -12,7 +12,7 @@ use OCP\AppFramework\Bootstrap\IBootContext;
|
|||||||
use OCP\AppFramework\Bootstrap\IBootstrap;
|
use OCP\AppFramework\Bootstrap\IBootstrap;
|
||||||
use OCP\AppFramework\Bootstrap\IRegistrationContext;
|
use OCP\AppFramework\Bootstrap\IRegistrationContext;
|
||||||
|
|
||||||
final class Application extends App implements IBootstrap {
|
class Application extends App implements IBootstrap {
|
||||||
public const APP_ID = 'ng_backup';
|
public const APP_ID = 'ng_backup';
|
||||||
|
|
||||||
public function __construct() {
|
public function __construct() {
|
||||||
@@ -20,8 +20,6 @@ final class Application extends App implements IBootstrap {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function register(IRegistrationContext $context): void {
|
public function register(IRegistrationContext $context): void {
|
||||||
$context->registerNotifierService(\OCA\NgBackup\Notification\Notifier::class);
|
|
||||||
$context->registerSetupCheck(\OCA\NgBackup\SetupCheck\BackupStatusCheck::class);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function boot(IBootContext $context): void {
|
public function boot(IBootContext $context): void {
|
||||||
|
|||||||
@@ -7,5 +7,5 @@ declare(strict_types=1);
|
|||||||
|
|
||||||
namespace OCA\NgBackup\Backend;
|
namespace OCA\NgBackup\Backend;
|
||||||
|
|
||||||
final class BackendException extends \RuntimeException {
|
class BackendException extends \RuntimeException {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,30 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Backend;
|
|
||||||
|
|
||||||
/** IBackend::getMany() for a backend where reading files one after the other costs nothing extra. */
|
|
||||||
trait GetsOneByOne {
|
|
||||||
/**
|
|
||||||
* @param list<string> $paths
|
|
||||||
* @return \Generator<string, string>
|
|
||||||
*/
|
|
||||||
public function getMany(array $paths): \Generator {
|
|
||||||
foreach ($paths as $path) {
|
|
||||||
$fh = $this->get($path);
|
|
||||||
try {
|
|
||||||
$data = stream_get_contents($fh);
|
|
||||||
} finally {
|
|
||||||
fclose($fh);
|
|
||||||
}
|
|
||||||
if ($data === false) {
|
|
||||||
throw new BackendException('Read failed for ' . $path);
|
|
||||||
}
|
|
||||||
yield $path => $data;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -22,40 +22,13 @@ interface IBackend {
|
|||||||
/** @return resource readable stream of the whole object */
|
/** @return resource readable stream of the whole object */
|
||||||
public function get(string $path);
|
public function get(string $path);
|
||||||
|
|
||||||
/**
|
|
||||||
* Many small objects whole, in the order given (the repository's index files). A backend
|
|
||||||
* where every request costs a round trip can fetch them several at a time.
|
|
||||||
*
|
|
||||||
* @param list<string> $paths
|
|
||||||
* @return \Generator<string, string> contents by path
|
|
||||||
* @throws BackendException for an object that cannot be read
|
|
||||||
*/
|
|
||||||
public function getMany(array $paths): \Generator;
|
|
||||||
|
|
||||||
/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
|
/** @return string bytes [$offset, $offset + $length) of the object (used to fetch single blobs from a pack) */
|
||||||
public function getRange(string $path, int $offset, int $length): string;
|
public function getRange(string $path, int $offset, int $length): string;
|
||||||
|
|
||||||
public function exists(string $path): bool;
|
public function exists(string $path): bool;
|
||||||
|
|
||||||
/**
|
|
||||||
* Move an object within the same location (used for the trash). Backends must use a native
|
|
||||||
* rename, not a read+write copy: some storages (e.g. SFTP) cannot reliably read and write on
|
|
||||||
* the same connection at once, since the read side is a lazily-pulled stream.
|
|
||||||
*/
|
|
||||||
public function move(string $from, string $to): void;
|
|
||||||
|
|
||||||
/** @return list<string> object paths below $prefix (recursive) */
|
/** @return list<string> object paths below $prefix (recursive) */
|
||||||
public function list(string $prefix): array;
|
public function list(string $prefix): array;
|
||||||
|
|
||||||
/**
|
|
||||||
* Object paths directly in $dir, for a folder that only holds objects (index/, snapshots/,
|
|
||||||
* catalog/, trash/info/). One listing, without checking the type of every entry: on a remote
|
|
||||||
* storage list() costs a round trip per entry, which for index/ (one file per pack) grows
|
|
||||||
* with the size of the repository.
|
|
||||||
*
|
|
||||||
* @return list<string>
|
|
||||||
*/
|
|
||||||
public function listFiles(string $dir): array;
|
|
||||||
|
|
||||||
public function delete(string $path): void;
|
public function delete(string $path): void;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,8 +8,6 @@ declare(strict_types=1);
|
|||||||
namespace OCA\NgBackup\Backend;
|
namespace OCA\NgBackup\Backend;
|
||||||
|
|
||||||
final class LocalBackend implements IBackend {
|
final class LocalBackend implements IBackend {
|
||||||
use GetsOneByOne;
|
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private string $root,
|
private string $root,
|
||||||
) {
|
) {
|
||||||
@@ -35,16 +33,11 @@ final class LocalBackend implements IBackend {
|
|||||||
if ($out === false) {
|
if ($out === false) {
|
||||||
throw new BackendException('Cannot write ' . $path);
|
throw new BackendException('Cannot write ' . $path);
|
||||||
}
|
}
|
||||||
$expected = StorageBackend::remaining($stream);
|
|
||||||
if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) {
|
if (stream_copy_to_stream($stream, $out) === false || !fflush($out)) {
|
||||||
throw new BackendException('Write failed for ' . $path);
|
throw new BackendException('Write failed for ' . $path);
|
||||||
}
|
}
|
||||||
fclose($out);
|
fclose($out);
|
||||||
$out = false;
|
$out = false;
|
||||||
clearstatcache(true, $tmp);
|
|
||||||
if ($expected !== null && filesize($tmp) !== $expected) {
|
|
||||||
throw new BackendException('Incomplete write for ' . $path);
|
|
||||||
}
|
|
||||||
if (!rename($tmp, $target)) {
|
if (!rename($tmp, $target)) {
|
||||||
throw new BackendException('Cannot finalise ' . $path);
|
throw new BackendException('Cannot finalise ' . $path);
|
||||||
}
|
}
|
||||||
@@ -101,7 +94,7 @@ final class LocalBackend implements IBackend {
|
|||||||
$result = [];
|
$result = [];
|
||||||
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS));
|
$it = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base, \FilesystemIterator::SKIP_DOTS));
|
||||||
foreach ($it as $file) {
|
foreach ($it as $file) {
|
||||||
if ($file->isFile() && !StorageBackend::isTemporary($file->getFilename())) {
|
if ($file->isFile() && !str_contains($file->getFilename(), '.part-')) {
|
||||||
$result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/');
|
$result[] = ltrim(substr($file->getPathname(), strlen($this->root)), '/');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -109,40 +102,10 @@ final class LocalBackend implements IBackend {
|
|||||||
return $result;
|
return $result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function listFiles(string $dir): array {
|
|
||||||
$base = $this->abs($dir);
|
|
||||||
if (!is_dir($base)) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
$result = [];
|
|
||||||
$names = scandir($base);
|
|
||||||
if ($names === false) {
|
|
||||||
throw new BackendException('Cannot list ' . $dir);
|
|
||||||
}
|
|
||||||
foreach ($names as $name) {
|
|
||||||
if ($name !== '.' && $name !== '..' && !StorageBackend::isTemporary($name)) {
|
|
||||||
$result[] = trim($dir, '/') . '/' . $name;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort($result);
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function delete(string $path): void {
|
public function delete(string $path): void {
|
||||||
@unlink($this->abs($path));
|
@unlink($this->abs($path));
|
||||||
}
|
}
|
||||||
|
|
||||||
public function move(string $from, string $to): void {
|
|
||||||
$target = $this->abs($to);
|
|
||||||
$dir = dirname($target);
|
|
||||||
if (!is_dir($dir) && !mkdir($dir, 0700, true) && !is_dir($dir)) {
|
|
||||||
throw new BackendException('Cannot create ' . $dir);
|
|
||||||
}
|
|
||||||
if (!rename($this->abs($from), $target)) {
|
|
||||||
throw new BackendException("Cannot move $from to $to");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function abs(string $path): string {
|
private function abs(string $path): string {
|
||||||
if (str_contains($path, '..') || str_starts_with($path, '/')) {
|
if (str_contains($path, '..') || str_starts_with($path, '/')) {
|
||||||
throw new BackendException('Invalid path: ' . $path);
|
throw new BackendException('Invalid path: ' . $path);
|
||||||
|
|||||||
@@ -1,628 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Backend;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Read stream for files_external's SFTP storage that keeps several READ requests in flight.
|
|
||||||
*
|
|
||||||
* files_external's own read stream asks for the next 256 KiB only after the previous one has
|
|
||||||
* arrived, so every request costs a full round trip: about 5 MB/s at 47 ms to a Hetzner Storage
|
|
||||||
* Box, whatever the line speed. With 8 requests in flight the same connection reached 15 MB/s.
|
|
||||||
* phpseclib's get() pipelines too, but only into memory or a local file, and only from offset 0.
|
|
||||||
*
|
|
||||||
* Talks to the storage's phpseclib connection through the same internal methods files_external's
|
|
||||||
* read stream uses (_send_sftp_packet, _get_sftp_packet, _realpath); open() returns null when the
|
|
||||||
* connection does not offer them (phpseclib 3 does not), and StorageBackend then falls back to the
|
|
||||||
* storage's own stream. A restore must rather be slow than fail: when a read fails halfway, the
|
|
||||||
* stream goes on from the same position with the storage's own stream. After MAX_FAILURES streams
|
|
||||||
* in a row failed, open() returns null for the rest of the process; one that works resets the count.
|
|
||||||
* Like that stream, it owns the connection while open: no other request may go over the same
|
|
||||||
* connection until it is closed (PackReaderGuard takes care of that for the repository).
|
|
||||||
*/
|
|
||||||
final class SftpReadStream {
|
|
||||||
private const PROTOCOL = 'ngbsftpread';
|
|
||||||
/** Requests in flight once a stream reads sequentially; it starts at one and doubles. */
|
|
||||||
public const DEPTH = 8;
|
|
||||||
/** Files read at once by readMany(): each holds a handle on the server while its batch runs. */
|
|
||||||
public const BATCH = 32;
|
|
||||||
/** Larger files are left to the usual read by readMany(), which holds every file in memory. */
|
|
||||||
private const MANY_MAX_BYTES = 16 * 1048576;
|
|
||||||
/** OpenSSH and the Hetzner Storage Box both answer at most 255 KiB per READ. */
|
|
||||||
private const CHUNK = 261120;
|
|
||||||
/** Every SFTP server must handle packets of this size (draft-ietf-secsh-filexfer-02, section 3). */
|
|
||||||
private const MIN_CHUNK = 32768;
|
|
||||||
|
|
||||||
private const FXP_OPEN = 3;
|
|
||||||
private const FXP_CLOSE = 4;
|
|
||||||
private const FXP_READ = 5;
|
|
||||||
private const FXP_STATUS = 101;
|
|
||||||
private const FXP_HANDLE = 102;
|
|
||||||
private const FXP_DATA = 103;
|
|
||||||
private const FXF_READ = 1;
|
|
||||||
private const FX_OK = 0;
|
|
||||||
private const FX_EOF = 1;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* READ size per connection, lowered when a server answers less than asked and then goes on, so
|
|
||||||
* later streams over that connection ask for no more than it gives.
|
|
||||||
*
|
|
||||||
* @var \WeakMap<object, int>|null
|
|
||||||
*/
|
|
||||||
private static ?\WeakMap $chunks = null;
|
|
||||||
private static int $requestId = 0x4e470000;
|
|
||||||
/**
|
|
||||||
* Connections whose server did not confirm closing a batch's files: readMany() is not used on
|
|
||||||
* them again, so handles cannot pile up.
|
|
||||||
*
|
|
||||||
* @var \WeakMap<object, true>|null
|
|
||||||
*/
|
|
||||||
private static ?\WeakMap $noBatches = null;
|
|
||||||
/** Streams in a row that failed; at MAX_FAILURES only the storage's own stream is used. */
|
|
||||||
private static int $failures = 0;
|
|
||||||
public const MAX_FAILURES = 3;
|
|
||||||
/** Set by stream_open() when the connection failed rather than the file. */
|
|
||||||
private static bool $openBroke = false;
|
|
||||||
/** Why the last stream_open() failed on the connection. */
|
|
||||||
private static string $openCause = '';
|
|
||||||
|
|
||||||
/** @var resource|null set by PHP for stream wrappers */
|
|
||||||
public $context;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The phpseclib connection. PHP creates stream wrappers without constructor arguments; it is
|
|
||||||
* set in stream_open(), and no other method is called when that fails.
|
|
||||||
*
|
|
||||||
* @psalm-suppress PropertyNotSetInConstructor
|
|
||||||
*/
|
|
||||||
private object $sftp;
|
|
||||||
/** The storage and the path on it, to fall back to the storage's own stream. */
|
|
||||||
private ?object $storage = null;
|
|
||||||
private string $file = '';
|
|
||||||
/** @var resource|null the storage's own stream, once this one has failed */
|
|
||||||
private $fallback = null;
|
|
||||||
private string $handle = '';
|
|
||||||
/** Position of the reader: the first byte of $buffer. */
|
|
||||||
private int $pos = 0;
|
|
||||||
/** Offset of the next READ to send. */
|
|
||||||
private int $next = 0;
|
|
||||||
private string $buffer = '';
|
|
||||||
/** @var list<array{int, int, int}> request id, offset, length, in the order sent */
|
|
||||||
private array $inFlight = [];
|
|
||||||
private int $depth = 1;
|
|
||||||
private bool $eof = false;
|
|
||||||
/** This stream gave up on its own requests. */
|
|
||||||
private bool $failed = false;
|
|
||||||
/** The connection is in an unknown state: nothing more is sent over it or read from it. */
|
|
||||||
private bool $broken = false;
|
|
||||||
/** Length of the last short answer, until the next answer tells whether it was the end of the file. */
|
|
||||||
private ?int $shortAnswer = null;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param object $storage files_external's SFTP storage (getConnection(), getRoot())
|
|
||||||
* @return resource|false|null the stream, false when the file cannot be opened, null when the
|
|
||||||
* connection does not offer what this stream needs
|
|
||||||
* @throws BackendException when the connection failed after the OPEN request went out: the
|
|
||||||
* storage's own stream must not be tried on it, as it would take a
|
|
||||||
* late answer to that request for its own
|
|
||||||
*/
|
|
||||||
public static function open(object $storage, string $path) {
|
|
||||||
if (self::$failures >= self::MAX_FAILURES) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$sftp = self::connection($storage);
|
|
||||||
if ($sftp === null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (!in_array(self::PROTOCOL, stream_get_wrappers(), true)) {
|
|
||||||
stream_wrapper_register(self::PROTOCOL, self::class);
|
|
||||||
}
|
|
||||||
$root = rtrim((string)$storage->getRoot(), '/');
|
|
||||||
$context = stream_context_create([self::PROTOCOL => [
|
|
||||||
'session' => $sftp,
|
|
||||||
'path' => $root . '/' . ltrim($path, '/'),
|
|
||||||
'storage' => $storage,
|
|
||||||
'file' => $path,
|
|
||||||
]]);
|
|
||||||
self::$openBroke = false;
|
|
||||||
$fh = @fopen(self::PROTOCOL . '://file', 'r', false, $context);
|
|
||||||
} catch (\Throwable) {
|
|
||||||
// nothing was sent yet
|
|
||||||
self::$failures++;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if ($fh === false && self::openBroke()) {
|
|
||||||
throw new BackendException('SFTP connection failed while opening ' . $path . ': ' . self::$openCause);
|
|
||||||
}
|
|
||||||
return $fh;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The storage's phpseclib connection, with its SFTP channel open, when it offers what these
|
|
||||||
* streams need; null otherwise.
|
|
||||||
*/
|
|
||||||
private static function connection(object $storage): ?object {
|
|
||||||
if (!is_callable([$storage, 'getConnection']) || !is_callable([$storage, 'getRoot'])) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$sftp = $storage->getConnection();
|
|
||||||
// is_callable and get_object_vars from here only see what is public: phpseclib 3 has
|
|
||||||
// methods of these names neither public nor at all.
|
|
||||||
if (!is_object($sftp) || !is_callable([$sftp, '_send_sftp_packet']) || !is_callable([$sftp, '_get_sftp_packet'])
|
|
||||||
|| !is_callable([$sftp, '_realpath']) || !is_callable([$sftp, 'pwd'])
|
|
||||||
|| !array_key_exists('packet_type', get_object_vars($sftp))) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
// phpseclib opens the SFTP channel on its first SFTP call, not at login: on a fresh
|
|
||||||
// connection a packet sent directly would go nowhere. pwd() opens it like any call does.
|
|
||||||
if ($sftp->pwd() === false) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return $sftp;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Read many small files whole, BATCH at a time: the OPEN, READ and CLOSE requests of a batch
|
|
||||||
* all go out before their answers are read, so a batch costs a few round trips instead of
|
|
||||||
* four per file. Meant for files of up to a few hundred KiB (the repository's index files):
|
|
||||||
* every file is held in memory.
|
|
||||||
*
|
|
||||||
* @param list<string> $paths below the storage's root
|
|
||||||
* @return array<string, string|null>|null contents by path; null for a file this could not
|
|
||||||
* read (missing, or a server error), so the caller
|
|
||||||
* reads it the usual way; null for all when the
|
|
||||||
* connection does not offer what this needs
|
|
||||||
* @throws BackendException when the connection failed while requests were out: the storage's
|
|
||||||
* own stream must not be tried on it
|
|
||||||
*/
|
|
||||||
public static function readMany(object $storage, array $paths): ?array {
|
|
||||||
if (self::$failures >= self::MAX_FAILURES) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$sftp = self::connection($storage);
|
|
||||||
$root = $sftp === null ? '' : rtrim((string)$storage->getRoot(), '/');
|
|
||||||
} catch (\Throwable) {
|
|
||||||
self::$failures++;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if ($sftp === null || self::$noBatches?->offsetExists($sftp) === true) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$result = array_fill_keys($paths, null);
|
|
||||||
foreach (array_chunk($paths, self::BATCH) as $batch) {
|
|
||||||
try {
|
|
||||||
[$read, $closed] = self::readBatch($sftp, $root, $batch);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
self::$failures++;
|
|
||||||
// Answers to requests already sent may still arrive, and files_external would take
|
|
||||||
// them for its own: end the connection rather than leave it out of step.
|
|
||||||
self::disconnect($sftp);
|
|
||||||
throw $e instanceof BackendException ? $e : new BackendException('SFTP connection failed while reading ' . count($batch) . ' files: ' . $e->getMessage(), 0, $e);
|
|
||||||
}
|
|
||||||
foreach ($read as $path => $contents) {
|
|
||||||
$result[$path] = $contents;
|
|
||||||
}
|
|
||||||
if (!$closed) {
|
|
||||||
// The server did not confirm closing these files: no more batches on this
|
|
||||||
// connection, so handles cannot pile up. The rest is read the usual way.
|
|
||||||
self::noMoreBatches($sftp);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
self::$failures = 0;
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function noMoreBatches(object $sftp): void {
|
|
||||||
self::$noBatches ??= new \WeakMap();
|
|
||||||
self::$noBatches[$sftp] = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function disconnect(object $sftp): void {
|
|
||||||
try {
|
|
||||||
if (is_callable([$sftp, 'disconnect'])) {
|
|
||||||
$sftp->disconnect();
|
|
||||||
}
|
|
||||||
} catch (\Throwable) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Requests of one kind for a whole batch, then their answers in order. @return list<array{int, string}> */
|
|
||||||
private static function exchange(object $sftp, int $type, array $packets): array {
|
|
||||||
$ids = [];
|
|
||||||
foreach ($packets as $packet) {
|
|
||||||
$id = self::nextId();
|
|
||||||
if (!$sftp->_send_sftp_packet($type, $packet, $id)) {
|
|
||||||
throw new BackendException('SFTP connection failed while sending a request');
|
|
||||||
}
|
|
||||||
$ids[] = $id;
|
|
||||||
}
|
|
||||||
$answers = [];
|
|
||||||
foreach ($ids as $id) {
|
|
||||||
$response = $sftp->_get_sftp_packet($id);
|
|
||||||
if (!is_string($response)) {
|
|
||||||
throw new BackendException('SFTP connection failed while waiting for an answer');
|
|
||||||
}
|
|
||||||
$answers[] = [$sftp->packet_type, $response];
|
|
||||||
}
|
|
||||||
return $answers;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param list<string> $batch
|
|
||||||
* @return array{0: array<string, string>, 1: bool} the files read whole, and whether the
|
|
||||||
* server confirmed closing every handle
|
|
||||||
*/
|
|
||||||
private static function readBatch(object $sftp, string $root, array $batch): array {
|
|
||||||
$opens = [];
|
|
||||||
$openPaths = [];
|
|
||||||
foreach ($batch as $path) {
|
|
||||||
$remote = $sftp->_realpath($root . '/' . ltrim($path, '/'));
|
|
||||||
if (is_string($remote)) {
|
|
||||||
$opens[] = pack('Na*N2', strlen($remote), $remote, self::FXF_READ, 0);
|
|
||||||
$openPaths[] = $path;
|
|
||||||
}
|
|
||||||
// a path that cannot be resolved is left to the caller's usual read
|
|
||||||
}
|
|
||||||
$handles = []; // path => handle
|
|
||||||
foreach (self::exchange($sftp, self::FXP_OPEN, $opens) as $i => [$type, $response]) {
|
|
||||||
if ($type === self::FXP_HANDLE) {
|
|
||||||
$handles[$openPaths[$i]] = substr($response, 4);
|
|
||||||
}
|
|
||||||
// any other answer (missing, no permission): the caller reads it the usual way
|
|
||||||
}
|
|
||||||
$data = array_fill_keys(array_keys($handles), '');
|
|
||||||
$reading = $handles;
|
|
||||||
while ($reading !== []) {
|
|
||||||
$length = self::chunk($sftp);
|
|
||||||
$reads = [];
|
|
||||||
foreach ($reading as $path => $handle) {
|
|
||||||
$offset = strlen($data[$path]);
|
|
||||||
$reads[] = pack('Na*N3', strlen($handle), $handle, intdiv($offset, 4294967296), $offset % 4294967296, $length);
|
|
||||||
}
|
|
||||||
$paths = array_keys($reading);
|
|
||||||
foreach (self::exchange($sftp, self::FXP_READ, $reads) as $i => [$type, $response]) {
|
|
||||||
$path = $paths[$i];
|
|
||||||
$declared = strlen($response) >= 4 ? unpack('N', $response) : false;
|
|
||||||
if ($type === self::FXP_DATA && is_array($declared) && $declared[1] > 0 && $declared[1] === strlen($response) - 4
|
|
||||||
&& $declared[1] <= $length && strlen($data[$path]) + $declared[1] <= self::MANY_MAX_BYTES) {
|
|
||||||
$data[$path] .= substr($response, 4); // more may follow: read on from there
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
unset($reading[$path]);
|
|
||||||
$status = $type === self::FXP_STATUS && is_array($declared) ? $declared[1] : null;
|
|
||||||
if ($status !== self::FX_EOF) {
|
|
||||||
// a server error, a malformed answer or a file too large for this: the caller
|
|
||||||
// reads it the usual way
|
|
||||||
unset($data[$path]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$closes = [];
|
|
||||||
foreach ($handles as $handle) {
|
|
||||||
$closes[] = pack('Na*', strlen($handle), $handle);
|
|
||||||
}
|
|
||||||
$closed = true;
|
|
||||||
foreach (self::exchange($sftp, self::FXP_CLOSE, $closes) as [$type, $response]) {
|
|
||||||
$status = $type === self::FXP_STATUS && strlen($response) >= 4 ? unpack('N', $response) : false;
|
|
||||||
if (!is_array($status) || $status[1] !== self::FX_OK) {
|
|
||||||
$closed = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return [$data, $closed];
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Test hook: forget the learned READ sizes and earlier failures. */
|
|
||||||
public static function reset(): void {
|
|
||||||
self::$chunks = null;
|
|
||||||
self::$noBatches = null;
|
|
||||||
self::$failures = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function stream_open(string $url, string $mode, int $options, ?string &$openedPath): bool {
|
|
||||||
$options = stream_context_get_options($this->context ?? stream_context_get_default())[self::PROTOCOL] ?? [];
|
|
||||||
if (!isset($options['session'], $options['path']) || !is_object($options['session'])) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
$this->sftp = $options['session'];
|
|
||||||
$storage = $options['storage'] ?? null;
|
|
||||||
$this->storage = is_object($storage) ? $storage : null;
|
|
||||||
$this->file = (string)($options['file'] ?? '');
|
|
||||||
try {
|
|
||||||
$remote = $this->sftp->_realpath((string)$options['path']);
|
|
||||||
if (!is_string($remote)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
$id = self::nextId();
|
|
||||||
if (!$this->sftp->_send_sftp_packet(self::FXP_OPEN, pack('Na*N2', strlen($remote), $remote, self::FXF_READ, 0), $id)) {
|
|
||||||
return $this->connectionFailed('the request could not be sent');
|
|
||||||
}
|
|
||||||
$response = $this->sftp->_get_sftp_packet($id);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
return $this->connectionFailed(get_class($e) . ': ' . $e->getMessage());
|
|
||||||
}
|
|
||||||
if (!is_string($response)) {
|
|
||||||
return $this->connectionFailed('no answer');
|
|
||||||
}
|
|
||||||
if ($this->sftp->packet_type !== self::FXP_HANDLE) {
|
|
||||||
return false; // an answer, but no handle: the file cannot be opened
|
|
||||||
}
|
|
||||||
$this->handle = substr($response, 4);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Not a missing file but a connection that does not behave as expected. */
|
|
||||||
private function connectionFailed(string $cause): bool {
|
|
||||||
self::$failures++;
|
|
||||||
self::$openBroke = true;
|
|
||||||
self::$openCause = $cause . self::connectionState($this->sftp);
|
|
||||||
// unknown state: closed, so that nothing else uses it (and the backend can build a new one)
|
|
||||||
self::disconnect($this->sftp);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** What phpseclib knows about the connection, for an error message. */
|
|
||||||
private static function connectionState(object $sftp): string {
|
|
||||||
$state = [];
|
|
||||||
try {
|
|
||||||
if (is_callable([$sftp, 'isConnected'])) {
|
|
||||||
$state[] = 'connected: ' . ($sftp->isConnected() ? 'yes' : 'no');
|
|
||||||
}
|
|
||||||
if (is_callable([$sftp, 'getLastError'])) {
|
|
||||||
$last = (string)$sftp->getLastError();
|
|
||||||
if ($last !== '') {
|
|
||||||
$state[] = 'ssh: ' . $last;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (is_callable([$sftp, 'getLastSFTPError'])) {
|
|
||||||
$last = (string)$sftp->getLastSFTPError();
|
|
||||||
if ($last !== '') {
|
|
||||||
$state[] = 'sftp: ' . $last;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (\Throwable) {
|
|
||||||
}
|
|
||||||
return $state === [] ? '' : ' (' . implode('; ', $state) . ')';
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Whether the last stream_open() failed on the connection; a call, as fopen() sets it. */
|
|
||||||
private static function openBroke(): bool {
|
|
||||||
return self::$openBroke;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function stream_read(int $count): string|false {
|
|
||||||
if ($this->fallback !== null) {
|
|
||||||
return fread($this->fallback, $count);
|
|
||||||
}
|
|
||||||
while (strlen($this->buffer) < $count && !$this->eof) {
|
|
||||||
if (!$this->receive()) {
|
|
||||||
return $this->useFallback() ? $this->stream_read($count) : false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$data = substr($this->buffer, 0, $count);
|
|
||||||
$this->buffer = substr($this->buffer, strlen($data));
|
|
||||||
$this->pos += strlen($data);
|
|
||||||
return $data;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function stream_eof(): bool {
|
|
||||||
if ($this->fallback !== null) {
|
|
||||||
return feof($this->fallback);
|
|
||||||
}
|
|
||||||
return $this->eof && $this->buffer === '';
|
|
||||||
}
|
|
||||||
|
|
||||||
public function stream_tell(): int {
|
|
||||||
if ($this->fallback !== null) {
|
|
||||||
$pos = ftell($this->fallback);
|
|
||||||
return $pos === false ? $this->pos : $pos;
|
|
||||||
}
|
|
||||||
return $this->pos;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function stream_seek(int $offset, int $whence = SEEK_SET): bool {
|
|
||||||
if ($this->fallback !== null) {
|
|
||||||
return fseek($this->fallback, $offset, $whence) === 0;
|
|
||||||
}
|
|
||||||
if ($whence === SEEK_CUR) {
|
|
||||||
$offset += $this->pos;
|
|
||||||
} elseif ($whence !== SEEK_SET) {
|
|
||||||
return false; // the size is never asked for, to save a round trip
|
|
||||||
}
|
|
||||||
if ($offset < 0) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if ($this->failed) {
|
|
||||||
$this->pos = $offset;
|
|
||||||
return $this->useFallback();
|
|
||||||
}
|
|
||||||
if ($offset >= $this->pos && $offset - $this->pos <= strlen($this->buffer)) {
|
|
||||||
$this->buffer = substr($this->buffer, $offset - $this->pos);
|
|
||||||
} else {
|
|
||||||
$this->drain();
|
|
||||||
$this->buffer = '';
|
|
||||||
$this->next = $offset;
|
|
||||||
$this->eof = false;
|
|
||||||
$this->depth = 1;
|
|
||||||
$this->shortAnswer = null;
|
|
||||||
}
|
|
||||||
$this->pos = $offset;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return array<int|string, int> */
|
|
||||||
public function stream_stat(): array {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
public function stream_close(): void {
|
|
||||||
if (!$this->failed) {
|
|
||||||
self::$failures = 0;
|
|
||||||
}
|
|
||||||
$fallback = $this->fallback;
|
|
||||||
$this->fallback = null;
|
|
||||||
if ($fallback !== null) {
|
|
||||||
fclose($fallback);
|
|
||||||
}
|
|
||||||
$this->closeHandle();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Go on with the storage's own stream from the current position, after a server error on this
|
|
||||||
* one. The data still buffered is dropped and read again. False after a connection error, or
|
|
||||||
* when that stream cannot be opened or positioned: then the read fails as it would have
|
|
||||||
* without this class. Reopening by path is safe for a repository: packs and index files are
|
|
||||||
* never rewritten, and every object is authenticated when it is decrypted.
|
|
||||||
*/
|
|
||||||
private function useFallback(): bool {
|
|
||||||
$this->closeHandle();
|
|
||||||
$this->buffer = '';
|
|
||||||
// files_external's stream takes whatever answer comes next on the connection: after a
|
|
||||||
// connection error, answers to this stream's requests may still be on their way.
|
|
||||||
if ($this->broken || $this->storage === null || !is_callable([$this->storage, 'fopen'])) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$fh = $this->storage->fopen($this->file, 'r');
|
|
||||||
if (!is_resource($fh)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if ($this->pos > 0 && fseek($fh, $this->pos) !== 0) {
|
|
||||||
fclose($fh);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
} catch (\Throwable) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
$this->fallback = $fh;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function closeHandle(): void {
|
|
||||||
$this->drain();
|
|
||||||
if ($this->handle !== '' && !$this->broken) {
|
|
||||||
try {
|
|
||||||
$id = self::nextId();
|
|
||||||
if (!$this->sftp->_send_sftp_packet(self::FXP_CLOSE, pack('Na*', strlen($this->handle), $this->handle), $id)
|
|
||||||
|| !is_string($this->sftp->_get_sftp_packet($id))) {
|
|
||||||
$this->broken = true;
|
|
||||||
}
|
|
||||||
} catch (\Throwable) {
|
|
||||||
$this->broken = true;
|
|
||||||
}
|
|
||||||
if ($this->broken) {
|
|
||||||
self::disconnect($this->sftp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$this->handle = '';
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Keep the pipeline full, then take the oldest answer. False on a connection or server error. */
|
|
||||||
private function receive(): bool {
|
|
||||||
if ($this->failed) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return $this->receiveOne();
|
|
||||||
} catch (\Throwable) {
|
|
||||||
$this->broken = true;
|
|
||||||
return $this->fail();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function receiveOne(): bool {
|
|
||||||
while (count($this->inFlight) < $this->depth) {
|
|
||||||
$id = self::nextId();
|
|
||||||
$length = self::chunk($this->sftp);
|
|
||||||
$packet = pack('Na*N3', strlen($this->handle), $this->handle, intdiv($this->next, 4294967296), $this->next % 4294967296, $length);
|
|
||||||
if (!$this->sftp->_send_sftp_packet(self::FXP_READ, $packet, $id)) {
|
|
||||||
$this->broken = true;
|
|
||||||
return $this->fail();
|
|
||||||
}
|
|
||||||
$this->inFlight[] = [$id, $this->next, $length];
|
|
||||||
$this->next += $length;
|
|
||||||
}
|
|
||||||
[$id, $offset, $length] = array_shift($this->inFlight);
|
|
||||||
$response = $this->sftp->_get_sftp_packet($id);
|
|
||||||
if (!is_string($response)) {
|
|
||||||
$this->broken = true;
|
|
||||||
return $this->fail();
|
|
||||||
}
|
|
||||||
if ($this->sftp->packet_type === self::FXP_DATA) {
|
|
||||||
$data = substr($response, 4);
|
|
||||||
if ($data === '') {
|
|
||||||
return $this->fail();
|
|
||||||
}
|
|
||||||
$this->buffer .= $data;
|
|
||||||
$this->depth = min(self::DEPTH, $this->depth * 2);
|
|
||||||
if ($this->shortAnswer !== null) {
|
|
||||||
// More data after a short answer: that one was the server's cap, not the end.
|
|
||||||
// Never below what every server must support: a one-off small answer is no cap.
|
|
||||||
self::$chunks ??= new \WeakMap();
|
|
||||||
self::$chunks[$this->sftp] = max(self::MIN_CHUNK, min(self::chunk($this->sftp), $this->shortAnswer));
|
|
||||||
$this->shortAnswer = null;
|
|
||||||
}
|
|
||||||
if (strlen($data) < $length) {
|
|
||||||
// Fewer bytes than asked: the end of the file, or a server that caps a READ. The
|
|
||||||
// requests behind this one assumed a full answer, so drop them and ask again from here.
|
|
||||||
$this->drain();
|
|
||||||
$this->next = $offset + strlen($data);
|
|
||||||
$this->shortAnswer = strlen($data);
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
$status = $this->sftp->packet_type === self::FXP_STATUS ? unpack('N', $response) : false;
|
|
||||||
if (is_array($status) && ($status[1] ?? null) === self::FX_EOF) {
|
|
||||||
$this->eof = true;
|
|
||||||
$this->shortAnswer = null;
|
|
||||||
$this->drain();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return $this->fail();
|
|
||||||
}
|
|
||||||
|
|
||||||
private function fail(): bool {
|
|
||||||
$this->drain();
|
|
||||||
if (!$this->failed) {
|
|
||||||
self::$failures++;
|
|
||||||
}
|
|
||||||
$this->failed = true;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Read and drop the answers still on their way, so the connection is free again. */
|
|
||||||
private function drain(): void {
|
|
||||||
$inFlight = $this->inFlight;
|
|
||||||
$this->inFlight = [];
|
|
||||||
try {
|
|
||||||
foreach ($this->broken ? [] : $inFlight as [$id]) {
|
|
||||||
if (!is_string($this->sftp->_get_sftp_packet($id))) {
|
|
||||||
$this->broken = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (\Throwable) {
|
|
||||||
$this->broken = true;
|
|
||||||
}
|
|
||||||
if ($this->broken) {
|
|
||||||
// unknown state: closed, so that nothing else uses it (and the backend can build a new one)
|
|
||||||
self::disconnect($this->sftp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function chunk(object $sftp): int {
|
|
||||||
$learned = self::$chunks?->offsetExists($sftp) === true ? self::$chunks->offsetGet($sftp) : null;
|
|
||||||
return is_int($learned) ? $learned : self::CHUNK;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function nextId(): int {
|
|
||||||
self::$requestId = self::$requestId >= 0x7fffffff ? 0x4e470000 : self::$requestId + 1;
|
|
||||||
return self::$requestId;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -17,43 +17,20 @@ use OCP\Files\Storage\IWriteStreamStorage;
|
|||||||
final class StorageBackend implements IBackend {
|
final class StorageBackend implements IBackend {
|
||||||
/** Object stores write whole objects atomically; other storages get a temp name + rename. */
|
/** Object stores write whole objects atomically; other storages get a temp name + rename. */
|
||||||
private bool $atomicPut;
|
private bool $atomicPut;
|
||||||
/** SFTP: read through SftpReadStream, which keeps several requests in flight. */
|
|
||||||
private bool $sftpReads;
|
|
||||||
/** Seconds without a request after which an SFTP connection is checked before it is used. */
|
|
||||||
public const IDLE_CHECK = 60;
|
|
||||||
private float $lastUsed;
|
|
||||||
/** Times the storage was built anew after its connection died (for tests and diagnostics). */
|
|
||||||
public int $reconnects = 0;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param (\Closure():IStorage)|null $reconnect builds the same storage anew, with a new
|
|
||||||
* connection
|
|
||||||
*/
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private IStorage $storage,
|
private IStorage $storage,
|
||||||
private string $base = '',
|
private string $base = '',
|
||||||
bool $createBase = true,
|
|
||||||
private ?\Closure $reconnect = null,
|
|
||||||
) {
|
) {
|
||||||
$this->lastUsed = microtime(true);
|
|
||||||
$this->base = trim($base, '/');
|
$this->base = trim($base, '/');
|
||||||
self::validate($this->base, true);
|
self::validate($this->base, true);
|
||||||
$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
|
$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
|
||||||
$this->sftpReads = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\SFTP::class);
|
if ($this->base !== '') {
|
||||||
if ($createBase && $this->base !== '') {
|
|
||||||
$this->mkdirs($this->base);
|
$this->mkdirs($this->base);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A temporary upload of put(): "<name>.part-<8 hex>". */
|
|
||||||
public static function isTemporary(string $name): bool {
|
|
||||||
return preg_match('/\.part-[0-9a-f]{8}$/', $name) === 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function put(string $path, $stream): void {
|
public function put(string $path, $stream): void {
|
||||||
// not repeated on a new connection: the stream is (partly) read
|
|
||||||
$this->idleCheck();
|
|
||||||
$expected = self::remaining($stream);
|
|
||||||
$target = $this->abs($path);
|
$target = $this->abs($path);
|
||||||
$this->mkdirs(dirname($target));
|
$this->mkdirs(dirname($target));
|
||||||
$writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4));
|
$writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4));
|
||||||
@@ -77,10 +54,6 @@ final class StorageBackend implements IBackend {
|
|||||||
if (is_resource($stream)) {
|
if (is_resource($stream)) {
|
||||||
fclose($stream);
|
fclose($stream);
|
||||||
}
|
}
|
||||||
// Some adapters report success after a short write or a failed flush: check the size.
|
|
||||||
if ($expected !== null && $this->storage->filesize($writeTo) !== $expected) {
|
|
||||||
throw new BackendException('Incomplete write for ' . $path);
|
|
||||||
}
|
|
||||||
if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) {
|
if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) {
|
||||||
throw new BackendException('Cannot finalise ' . $path);
|
throw new BackendException('Cannot finalise ' . $path);
|
||||||
}
|
}
|
||||||
@@ -96,19 +69,6 @@ final class StorageBackend implements IBackend {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function get(string $path) {
|
public function get(string $path) {
|
||||||
return $this->call(fn () => $this->open($path));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return resource */
|
|
||||||
private function open(string $path) {
|
|
||||||
if ($this->sftpReads) {
|
|
||||||
$fh = SftpReadStream::open($this->storage, $this->abs($path));
|
|
||||||
if (is_resource($fh)) {
|
|
||||||
return $fh;
|
|
||||||
}
|
|
||||||
// null: not supported or failing for now; false: not opened, which need not mean
|
|
||||||
// missing. Either way the storage's own stream decides.
|
|
||||||
}
|
|
||||||
$fh = $this->storage->fopen($this->abs($path), 'r');
|
$fh = $this->storage->fopen($this->abs($path), 'r');
|
||||||
if ($fh === false) {
|
if ($fh === false) {
|
||||||
throw new BackendException('Not found: ' . $path);
|
throw new BackendException('Not found: ' . $path);
|
||||||
@@ -116,40 +76,7 @@ final class StorageBackend implements IBackend {
|
|||||||
return $fh;
|
return $fh;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Over SFTP several at a time (SftpReadStream::readMany()); elsewhere one after the other. */
|
|
||||||
public function getMany(array $paths): \Generator {
|
|
||||||
foreach (array_chunk($paths, 8 * SftpReadStream::BATCH) as $chunk) {
|
|
||||||
$fast = null;
|
|
||||||
if ($this->sftpReads) {
|
|
||||||
$abs = array_map(fn (string $p): string => $this->abs($p), $chunk);
|
|
||||||
$fast = $this->call(fn () => SftpReadStream::readMany($this->storage, $abs));
|
|
||||||
$fast = $fast === null ? null : array_combine($chunk, array_map(fn (string $a): ?string => $fast[$a] ?? null, $abs));
|
|
||||||
}
|
|
||||||
foreach ($chunk as $path) {
|
|
||||||
$data = $fast[$path] ?? null;
|
|
||||||
if ($data === null) {
|
|
||||||
// not read in the batch (or no batches here): the usual way, which also
|
|
||||||
// reports a missing file as such
|
|
||||||
$fh = $this->get($path);
|
|
||||||
try {
|
|
||||||
$data = stream_get_contents($fh);
|
|
||||||
} finally {
|
|
||||||
fclose($fh);
|
|
||||||
}
|
|
||||||
if ($data === false) {
|
|
||||||
throw new BackendException('Read failed for ' . $path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
yield $path => $data;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getRange(string $path, int $offset, int $length): string {
|
public function getRange(string $path, int $offset, int $length): string {
|
||||||
return $this->call(fn () => $this->readRange($path, $offset, $length));
|
|
||||||
}
|
|
||||||
|
|
||||||
private function readRange(string $path, int $offset, int $length): string {
|
|
||||||
$fh = $this->get($path);
|
$fh = $this->get($path);
|
||||||
try {
|
try {
|
||||||
if (fseek($fh, $offset) !== 0) {
|
if (fseek($fh, $offset) !== 0) {
|
||||||
@@ -172,213 +99,22 @@ final class StorageBackend implements IBackend {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Whether the storage accepts the connection settings (files_external's own check). */
|
|
||||||
public function canConnect(): bool {
|
|
||||||
try {
|
|
||||||
return !method_exists($this->storage, 'test') || $this->storage->test();
|
|
||||||
} catch (\Throwable) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function exists(string $path): bool {
|
public function exists(string $path): bool {
|
||||||
$abs = $this->abs($path);
|
return $this->storage->file_exists($this->abs($path));
|
||||||
return $this->call(fn () => $this->storage->file_exists($abs), fn (bool $found) => !$found);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function list(string $prefix): array {
|
public function list(string $prefix): array {
|
||||||
$abs = $this->abs($prefix);
|
$result = [];
|
||||||
$result = $this->call(function () use ($abs): array {
|
$this->listInto($this->abs($prefix), $result);
|
||||||
$result = [];
|
|
||||||
$this->listInto($abs, $result);
|
|
||||||
return $result;
|
|
||||||
}, fn (array $found) => $found === []);
|
|
||||||
sort($result);
|
sort($result);
|
||||||
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
||||||
return array_map(fn ($p) => substr($p, $cut), $result);
|
return array_map(fn ($p) => substr($p, $cut), $result);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function delete(string $path): void {
|
public function delete(string $path): void {
|
||||||
$this->idleCheck();
|
|
||||||
$this->storage->unlink($this->abs($path));
|
$this->storage->unlink($this->abs($path));
|
||||||
}
|
}
|
||||||
|
|
||||||
public function move(string $from, string $to): void {
|
|
||||||
// not repeated on a new connection: the server may have done it before the connection died
|
|
||||||
$this->idleCheck();
|
|
||||||
$target = $this->abs($to);
|
|
||||||
$this->mkdirs(dirname($target));
|
|
||||||
if (!$this->storage->rename($this->abs($from), $target)) {
|
|
||||||
throw new BackendException("Cannot move $from to $to");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function listFiles(string $dir): array {
|
|
||||||
$abs = $this->abs($dir);
|
|
||||||
return $this->call(fn () => $this->filesIn($dir, $abs), fn (array $found) => $found === []);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return list<string> */
|
|
||||||
private function filesIn(string $dir, string $abs): array {
|
|
||||||
if (!$this->storage->is_dir($abs)) {
|
|
||||||
return []; // not created yet
|
|
||||||
}
|
|
||||||
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
|
|
||||||
$names = $this->smbNames($abs);
|
|
||||||
if ($names !== null) {
|
|
||||||
$result = [];
|
|
||||||
foreach ($names as $name) {
|
|
||||||
if (!self::isTemporary($name)) {
|
|
||||||
$result[] = substr($abs . '/' . $name, $cut);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort($result);
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
$dh = $this->storage->opendir($abs);
|
|
||||||
if ($dh === false) {
|
|
||||||
// An existing folder read as empty would look like a new repository.
|
|
||||||
throw new BackendException('Cannot list ' . $dir);
|
|
||||||
}
|
|
||||||
$result = [];
|
|
||||||
while (($name = readdir($dh)) !== false) {
|
|
||||||
if ($name !== '.' && $name !== '..' && !self::isTemporary($name)) {
|
|
||||||
$result[] = substr($abs . '/' . $name, $cut);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
closedir($dh);
|
|
||||||
sort($result);
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The names in an SMB folder, read in one listing; null for other storages, with access
|
|
||||||
* control lists on, or when the storage's internals are not as expected. The SMB storage's
|
|
||||||
* opendir() asks the server for the attributes of each entry to leave out hidden ones: with
|
|
||||||
* thousands of index files that took minutes. Hidden entries are still left out: names
|
|
||||||
* starting with a dot, and, after asking, any name a repository does not use (letters,
|
|
||||||
* digits, - and _), such as desktop.ini or Thumbs.db. The repository's own files have such
|
|
||||||
* names: they are not checked one by one, which was the cost.
|
|
||||||
*
|
|
||||||
* @return list<string>|null
|
|
||||||
*/
|
|
||||||
private function smbNames(string $abs): ?array {
|
|
||||||
if (!$this->storage instanceof \OCA\Files_External\Lib\Storage\SMB) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
/** @var list<string>|null */
|
|
||||||
return (function (string $path): ?array {
|
|
||||||
/** @psalm-suppress UndefinedThisPropertyFetch */
|
|
||||||
if ($this->checkAcl) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$names = [];
|
|
||||||
/** @psalm-suppress UndefinedThisPropertyFetch, UndefinedMethod */
|
|
||||||
foreach ($this->share->dir(ltrim($this->buildPath($path), '/')) as $info) {
|
|
||||||
$name = (string)$info->getName();
|
|
||||||
/** @psalm-suppress UndefinedThisPropertyFetch */
|
|
||||||
if (!$this->showHidden && (str_starts_with($name, '.')
|
|
||||||
|| (preg_match('/^[A-Za-z0-9_-]+$/', $name) !== 1 && $info->isHidden()))) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$names[] = $name;
|
|
||||||
}
|
|
||||||
return $names;
|
|
||||||
})->call($this->storage, $abs);
|
|
||||||
} catch (\Throwable) {
|
|
||||||
return null; // another SMB storage version, or an error: the usual way reports it
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run $op; when it throws, or $failed says its result may come from a dead connection, and
|
|
||||||
* the SFTP connection turns out to be dead, run it once more on a new connection.
|
|
||||||
*
|
|
||||||
* @template T
|
|
||||||
* @param \Closure():T $op
|
|
||||||
* @param (\Closure(T):bool)|null $failed
|
|
||||||
* @return T
|
|
||||||
*/
|
|
||||||
private function call(\Closure $op, ?\Closure $failed = null): mixed {
|
|
||||||
$this->idleCheck();
|
|
||||||
try {
|
|
||||||
$result = $op();
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
if (!$this->reopenIfDead()) {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
return $op();
|
|
||||||
}
|
|
||||||
if ($failed !== null && $failed($result) && $this->reopenIfDead()) {
|
|
||||||
return $op();
|
|
||||||
}
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A Hetzner Storage Box drops an SFTP connection that carried a lot of data and then stayed
|
|
||||||
* quiet for a few minutes, while phpseclib still takes it for open; files_external never
|
|
||||||
* builds a new one, and every request on it fails as if the file were missing. So after
|
|
||||||
* such a pause the connection is checked (one round trip) before it is used again.
|
|
||||||
*/
|
|
||||||
private function idleCheck(): void {
|
|
||||||
$now = microtime(true);
|
|
||||||
if ($this->reconnect !== null && $this->sftpReads && $now - $this->lastUsed >= self::IDLE_CHECK
|
|
||||||
&& $this->alive(true) === false) {
|
|
||||||
$this->reopen();
|
|
||||||
}
|
|
||||||
$this->lastUsed = microtime(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function reopenIfDead(): bool {
|
|
||||||
if ($this->reconnect === null || !$this->sftpReads || $this->alive(false) !== false) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
$this->reopen();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Whether the storage's SFTP connection is alive; null when it cannot tell.
|
|
||||||
* $ask: send a request instead of trusting what phpseclib saw last.
|
|
||||||
*/
|
|
||||||
private function alive(bool $ask): ?bool {
|
|
||||||
try {
|
|
||||||
if (!is_callable([$this->storage, 'getConnection'])) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$sftp = $this->storage->getConnection();
|
|
||||||
if (!is_object($sftp) || !is_callable([$sftp, 'isConnected'])) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (!$sftp->isConnected()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!$ask || !is_callable([$sftp, 'stat'])) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (is_callable([$sftp, 'clearStatCache'])) {
|
|
||||||
$sftp->clearStatCache(); // or stat() answers from memory
|
|
||||||
}
|
|
||||||
$sftp->stat('.');
|
|
||||||
return (bool)$sftp->isConnected();
|
|
||||||
} catch (\Throwable) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function reopen(): void {
|
|
||||||
/** @var \Closure():IStorage $reconnect */
|
|
||||||
$reconnect = $this->reconnect;
|
|
||||||
$this->storage = $reconnect();
|
|
||||||
$this->reconnects++;
|
|
||||||
$this->lastUsed = microtime(true);
|
|
||||||
if (function_exists('OCP\Log\logger')) {
|
|
||||||
\OCP\Log\logger('ng_backup')->info('The SFTP connection to the backup location was lost; reconnected');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function listInto(string $dir, array &$result): void {
|
private function listInto(string $dir, array &$result): void {
|
||||||
if (!$this->storage->is_dir($dir)) {
|
if (!$this->storage->is_dir($dir)) {
|
||||||
return;
|
return;
|
||||||
@@ -388,39 +124,19 @@ final class StorageBackend implements IBackend {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
while (($name = readdir($dh)) !== false) {
|
while (($name = readdir($dh)) !== false) {
|
||||||
if ($name === '.' || $name === '..') {
|
if ($name === '.' || $name === '..' || str_contains($name, '.part-')) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
$child = $dir . '/' . $name;
|
$child = $dir . '/' . $name;
|
||||||
if ($this->storage->is_dir($child)) {
|
if ($this->storage->is_dir($child)) {
|
||||||
$this->listInto($child, $result);
|
$this->listInto($child, $result);
|
||||||
} elseif (!self::isTemporary($name)) {
|
} else {
|
||||||
$result[] = $child;
|
$result[] = $child;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
closedir($dh);
|
closedir($dh);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Bytes left to read in a local $stream (php://temp, php://memory or a plain file), or null:
|
|
||||||
* stream wrappers of remote storages do not report a reliable size.
|
|
||||||
*/
|
|
||||||
public static function remaining(mixed $stream): ?int {
|
|
||||||
if (!is_resource($stream)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$meta = stream_get_meta_data($stream);
|
|
||||||
if (!($meta['seekable'] ?? false) || !in_array($meta['wrapper_type'] ?? '', ['PHP', 'plainfile'], true)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$stat = fstat($stream);
|
|
||||||
$pos = ftell($stream);
|
|
||||||
if ($stat === false || $pos === false) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return max(0, $stat['size'] - $pos);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function mkdirs(string $dir): void {
|
private function mkdirs(string $dir): void {
|
||||||
if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) {
|
if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
@@ -22,12 +22,10 @@ use Psr\Log\LoggerInterface;
|
|||||||
* With system cron a step may take minutes; with webcron/AJAX cron it is kept short so it fits
|
* With system cron a step may take minutes; with webcron/AJAX cron it is kept short so it fits
|
||||||
* in a normal web request (hosted Nextcloud).
|
* in a normal web request (hosted Nextcloud).
|
||||||
*/
|
*/
|
||||||
final class BackupJob extends TimedJob {
|
class BackupJob extends TimedJob {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
ITimeFactory $time,
|
ITimeFactory $time,
|
||||||
private BackupService $backups,
|
private BackupService $backups,
|
||||||
private \OCA\NgBackup\Service\PruneService $prune,
|
|
||||||
private \OCA\NgBackup\Service\AlertService $alerts,
|
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RunMapper $runs,
|
private RunMapper $runs,
|
||||||
private IAppConfig $appConfig,
|
private IAppConfig $appConfig,
|
||||||
@@ -38,63 +36,15 @@ final class BackupJob extends TimedJob {
|
|||||||
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
|
$this->setTimeSensitivity(IJob::TIME_SENSITIVE);
|
||||||
}
|
}
|
||||||
|
|
||||||
private const MAX_DAILY_ATTEMPTS = 3;
|
|
||||||
|
|
||||||
protected function run($argument): void {
|
protected function run($argument): void {
|
||||||
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
|
$budget = PHP_SAPI === 'cli' ? 240.0 : 20.0;
|
||||||
$deadline = microtime(true) + $budget;
|
$deadline = microtime(true) + $budget;
|
||||||
$this->startScheduled();
|
$this->startScheduled();
|
||||||
foreach ($this->runs->findRunning(null, BackupService::KIND_FULL) as $run) {
|
foreach ($this->runs->findRunning(null, BackupService::KIND_FULL) as $run) {
|
||||||
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
|
while (microtime(true) < $deadline && $run->getStatus() === 'running') {
|
||||||
$next = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
$run = $this->backups->step($run, min($deadline, microtime(true) + $budget));
|
||||||
if ($next === $run) {
|
|
||||||
break; // step() hands back the same object when another process holds the run's lease
|
|
||||||
}
|
|
||||||
$run = $next;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$this->failStaleRestores();
|
|
||||||
$this->weeklyPrune();
|
|
||||||
foreach ($this->targets->list() as $t) {
|
|
||||||
$this->alerts->checkStale($t);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** A restore run that has not moved for 6 hours is considered dead (it would block cleanup forever). */
|
|
||||||
private function failStaleRestores(): void {
|
|
||||||
foreach ($this->runs->findRunning(null, 'restore') as $run) {
|
|
||||||
if (time() - $run->getUpdatedAt() > 6 * 3600) {
|
|
||||||
$run->setStatus(\OCA\NgBackup\Db\Run::FAILED);
|
|
||||||
$run->setError('No progress for 6 hours; marked as failed');
|
|
||||||
$run->setFinishedAt(time());
|
|
||||||
$this->runs->update($run);
|
|
||||||
$this->alerts->runFailed($run, '#' . $run->getTargetId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Once a week, apply the retention policy to non-append-only locations: from Sunday on, at the
|
|
||||||
* first moment no backup is running, so a long Sunday backup only delays it instead of skipping
|
|
||||||
* the week.
|
|
||||||
*/
|
|
||||||
private function weeklyPrune(): void {
|
|
||||||
$lastWeek = $this->appConfig->getValueString(Application::APP_ID, 'last_prune_week', '');
|
|
||||||
$due = date('N') === '7' ? date('o-W') : date('o-W', (int)strtotime('last sunday'));
|
|
||||||
if (strcmp($lastWeek, $due) >= 0 || $this->runs->findRunning(null, BackupService::KIND_FULL) !== []) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
foreach ($this->targets->list() as $t) {
|
|
||||||
if ($t->getAppendOnly()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$this->prune->apply($t, false, true);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$this->logger->warning('NG Backup: cleanup of ' . $t->getName() . ' failed: ' . $e->getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, 'last_prune_week', $due);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
|
/** Daily schedule: appconfig ng_backup/schedule_time "HH:MM" (server time); empty = no schedule. */
|
||||||
@@ -103,7 +53,7 @@ final class BackupJob extends TimedJob {
|
|||||||
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
|
if (!preg_match('/^([01]\d|2[0-3]):[0-5]\d$/', $at)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
$todayAt = (int)strtotime(date('Y-m-d') . ' ' . $at);
|
$todayAt = strtotime(date('Y-m-d') . ' ' . $at);
|
||||||
if (time() < $todayAt) {
|
if (time() < $todayAt) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -112,12 +62,6 @@ final class BackupJob extends TimedJob {
|
|||||||
if ($this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
|
if ($this->runs->findRunning($t->getId(), BackupService::KIND_FULL) !== [] || ($last !== null && $last->getStartedAt() >= $todayAt)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// After a failure: retry at most MAX_DAILY_ATTEMPTS times a day, an hour apart, instead
|
|
||||||
// of starting (and notifying about) a new full run every five minutes all day.
|
|
||||||
$today = $this->runs->startedSince($t->getId(), BackupService::KIND_FULL, $todayAt);
|
|
||||||
if (count($today) >= self::MAX_DAILY_ATTEMPTS || ($today !== [] && time() - (int)$today[0]->getFinishedAt() < 3600)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
$this->backups->start($t, 'scheduled');
|
$this->backups->start($t, 'scheduled');
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
|
|||||||
@@ -16,46 +16,29 @@ use OCP\BackgroundJob\QueuedJob;
|
|||||||
use Psr\Log\LoggerInterface;
|
use Psr\Log\LoggerInterface;
|
||||||
|
|
||||||
/** Runs a restore requested from the web interface, outside the web request. */
|
/** Runs a restore requested from the web interface, outside the web request. */
|
||||||
final class RestoreJob extends QueuedJob {
|
class RestoreJob extends QueuedJob {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
ITimeFactory $time,
|
ITimeFactory $time,
|
||||||
private RunMapper $runs,
|
private RunMapper $runs,
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RestoreService $restore,
|
private RestoreService $restore,
|
||||||
private LoggerInterface $logger,
|
private LoggerInterface $logger,
|
||||||
private \OCA\NgBackup\Service\AlertService $alerts,
|
|
||||||
) {
|
) {
|
||||||
parent::__construct($time);
|
parent::__construct($time);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function run($argument): void {
|
protected function run($argument): void {
|
||||||
$runId = (int)($argument['run'] ?? 0);
|
$run = $this->runs->find((int)$argument['run']);
|
||||||
try {
|
|
||||||
$run = $this->runs->find($runId);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$this->logger->error('NG Backup restore job: run ' . $runId . ' not found', ['exception' => $e]);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if ($run->getStatus() !== Run::RUNNING) {
|
if ($run->getStatus() !== Run::RUNNING) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
$req = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
$run->setPhase('restoring');
|
||||||
|
$run->setUpdatedAt(time());
|
||||||
|
$this->runs->update($run);
|
||||||
|
@set_time_limit(0);
|
||||||
try {
|
try {
|
||||||
$req = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
$result = $this->restore->restoreUserFiles($this->targets->get((string)$run->getTargetId()), $req['snapshot'], $req['path'], $req['mode']);
|
||||||
$run->setPhase('restoring');
|
|
||||||
$run->setUpdatedAt(time());
|
|
||||||
$run = $this->runs->update($run);
|
|
||||||
@set_time_limit(0);
|
|
||||||
$lastBeat = time();
|
|
||||||
$result = $this->restore->restoreUserFiles($this->targets->get((string)$run->getTargetId()), $req['snapshot'], $req['path'], $req['mode'],
|
|
||||||
function (int $done, int $total) use (&$run, &$lastBeat): void {
|
|
||||||
// Heartbeat, so a long restore is not mistaken for a dead one.
|
|
||||||
if (time() - $lastBeat >= 60) {
|
|
||||||
$lastBeat = time();
|
|
||||||
$run->setPhase("restoring $done/$total");
|
|
||||||
$run->setUpdatedAt(time());
|
|
||||||
$run = $this->runs->update($run);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
$run->setStatus(Run::DONE);
|
$run->setStatus(Run::DONE);
|
||||||
$run->setPhase('done');
|
$run->setPhase('done');
|
||||||
$run->setStats(json_encode($result, JSON_THROW_ON_ERROR));
|
$run->setStats(json_encode($result, JSON_THROW_ON_ERROR));
|
||||||
@@ -63,10 +46,6 @@ final class RestoreJob extends QueuedJob {
|
|||||||
$this->logger->error('NG Backup restore ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
$this->logger->error('NG Backup restore ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
||||||
$run->setStatus(Run::FAILED);
|
$run->setStatus(Run::FAILED);
|
||||||
$run->setError($e->getMessage());
|
$run->setError($e->getMessage());
|
||||||
try {
|
|
||||||
$this->alerts->runFailed($run, '#' . $run->getTargetId());
|
|
||||||
} catch (\Throwable) {
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
$run->setFinishedAt(time());
|
$run->setFinishedAt(time());
|
||||||
$run->setUpdatedAt(time());
|
$run->setUpdatedAt(time());
|
||||||
|
|||||||
@@ -1,64 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use OCA\NgBackup\Service\UserBackupService;
|
|
||||||
use OCA\NgBackup\UserMigration\UserMigrationInstaller;
|
|
||||||
use OCP\IUserManager;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class BackupUser extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private UserBackupService $userBackup,
|
|
||||||
private IUserManager $userManager,
|
|
||||||
private UserMigrationInstaller $installer,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:user:backup')
|
|
||||||
->setDescription('Back up one user via user_migration (account, settings, files and everything else a migrator covers)')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->addArgument('user', InputArgument::REQUIRED, 'User id')
|
|
||||||
->addOption('install-user-migration', null, InputOption::VALUE_NONE, 'Install and enable user_migration if it is missing (non-interactive consent)')
|
|
||||||
->setHelp(<<<'HELP'
|
|
||||||
Streams a user_migration export straight into the encrypted repository: no zip, no copy in the
|
|
||||||
user's own storage. Shares data with the regular backup; this needs the user_migration app
|
|
||||||
enabled. The manifest path this prints is what occ backup:user:restore expects.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
occ backup:user:backup offsite alice
|
|
||||||
HELP);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$uid = (string)$input->getArgument('user');
|
|
||||||
$user = $this->userManager->get($uid);
|
|
||||||
if ($user === null) {
|
|
||||||
$output->writeln("<error>No such user: $uid</error>");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$target = $this->targets->get($input->getArgument('target'));
|
|
||||||
$this->installer->ensureEnabled($input, $output);
|
|
||||||
$manifest = $this->userBackup->backupUser($target, $user);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$output->writeln("Backed up <info>$uid</info> to $manifest.");
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputArgument;
|
|||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class Browse extends Command {
|
class Browse extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RestoreService $restore,
|
private RestoreService $restore,
|
||||||
|
|||||||
@@ -1,74 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Db\Run;
|
|
||||||
use OCA\NgBackup\Db\RunMapper;
|
|
||||||
use OCA\NgBackup\Service\BackupService;
|
|
||||||
use OCP\AppFramework\Db\DoesNotExistException;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class CancelRun extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private RunMapper $runs,
|
|
||||||
private BackupService $backups,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:cancel')
|
|
||||||
->setDescription('Stop a running backup')
|
|
||||||
->addArgument('run', InputArgument::REQUIRED, 'Run number, as shown by backup:status')
|
|
||||||
->setHelp("No new step of the run starts; the step in progress finishes first (usually within a
|
|
||||||
"
|
|
||||||
. "minute), so a backup that is just being completed is not cut off. The process working on the
|
|
||||||
"
|
|
||||||
. "run (occ backup:run, or cron) then stops. A backup started by an earlier version is marked
|
|
||||||
"
|
|
||||||
. "cancelled after two minutes and stops at the end of its step. What it already stored is
|
|
||||||
"
|
|
||||||
. "removed by the next cleanup.");
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$id = (string)$input->getArgument('run');
|
|
||||||
if (!ctype_digit($id)) {
|
|
||||||
$output->writeln('<error>Give the run number, as shown by backup:status.</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$run = $this->runs->find((int)$id);
|
|
||||||
} catch (DoesNotExistException) {
|
|
||||||
$output->writeln("<error>There is no run $id.</error>");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
if ($run->getKind() !== BackupService::KIND_FULL) {
|
|
||||||
$output->writeln("<error>Run $id is not a backup; only backups can be cancelled.</error>");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
if ($run->getStatus() === Run::RUNNING) {
|
|
||||||
$output->writeln("Waiting for the current step of run $id to finish...");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$run = $this->backups->cancel($run);
|
|
||||||
} catch (\RuntimeException $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
if ($run->getStatus() !== Run::CANCELLED) {
|
|
||||||
$output->writeln("Run $id is not running (status: {$run->getStatus()}); nothing to cancel.");
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
$output->writeln("Run $id is cancelled.");
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,7 +14,7 @@ use Symfony\Component\Console\Input\InputOption;
|
|||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
use Symfony\Component\Console\Question\Question;
|
use Symfony\Component\Console\Question\Question;
|
||||||
|
|
||||||
final class KeyConfirm extends Command {
|
class KeyConfirm extends Command {
|
||||||
public const STATEMENT = 'I have downloaded the recovery kit and stored it safely, outside this server. '
|
public const STATEMENT = 'I have downloaded the recovery kit and stored it safely, outside this server. '
|
||||||
. 'I understand that keeping these keys safe is my own responsibility and that backups cannot be restored without them.';
|
. 'I understand that keeping these keys safe is my own responsibility and that backups cannot be restored without them.';
|
||||||
public const CONFIRMATION = 'Yes, I confirm';
|
public const CONFIRMATION = 'Yes, I confirm';
|
||||||
@@ -28,9 +28,7 @@ final class KeyConfirm extends Command {
|
|||||||
protected function configure(): void {
|
protected function configure(): void {
|
||||||
$this->setName('backup:key:confirm')
|
$this->setName('backup:key:confirm')
|
||||||
->setDescription('Confirm that the recovery kit is stored safely (required before the first backup)')
|
->setDescription('Confirm that the recovery kit is stored safely (required before the first backup)')
|
||||||
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Admin user id recorded with the confirmation', 'occ')
|
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Admin user id recorded with the confirmation', 'occ');
|
||||||
->addOption('code', null, InputOption::VALUE_REQUIRED, 'Confirmation code from the kit file (for scripts; asked otherwise)')
|
|
||||||
->addOption('confirm', null, InputOption::VALUE_REQUIRED, 'The confirmation phrase "' . self::CONFIRMATION . '" (for scripts; asked otherwise)');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
protected function execute(InputInterface $input, OutputInterface $output): int {
|
||||||
@@ -42,12 +40,7 @@ final class KeyConfirm extends Command {
|
|||||||
$output->writeln('');
|
$output->writeln('');
|
||||||
$output->writeln(wordwrap(self::STATEMENT, 78));
|
$output->writeln(wordwrap(self::STATEMENT, 78));
|
||||||
$output->writeln('');
|
$output->writeln('');
|
||||||
$code = $input->getOption('code') ?? (string)$this->getHelper('question')->ask($input, $output, new Question('Confirmation code from the recovery kit file ("confirmation_code"): '));
|
$answer = (string)$this->getHelper('question')->ask($input, $output, new Question('Type "' . self::CONFIRMATION . '" to proceed: '));
|
||||||
if (!$this->keys->checkKitCode($code)) {
|
|
||||||
$output->writeln('<error>That code does not match the current recovery kit (download it with occ backup:key:kit first).</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$answer = $input->getOption('confirm') ?? (string)$this->getHelper('question')->ask($input, $output, new Question('Type "' . self::CONFIRMATION . '" to proceed: '));
|
|
||||||
if (trim($answer) !== self::CONFIRMATION) {
|
if (trim($answer) !== self::CONFIRMATION) {
|
||||||
$output->writeln('<error>Not confirmed.</error>');
|
$output->writeln('<error>Not confirmed.</error>');
|
||||||
return 1;
|
return 1;
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\KeyService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
use Symfony\Component\Console\Question\Question;
|
|
||||||
|
|
||||||
final class KeyImport extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private KeyService $keys,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:key:import')
|
|
||||||
->setDescription('Disaster recovery: adopt the backup key from a recovery kit on a fresh installation')
|
|
||||||
->addArgument('kit-file', InputArgument::REQUIRED, 'The recovery kit file (occ backup:key:kit on the original installation)')
|
|
||||||
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking')
|
|
||||||
->setHelp(<<<'HELP'
|
|
||||||
For a fresh installation only (occ backup:key:init must not have been run here yet). Reads one
|
|
||||||
of the kit's passphrase slots to recover the original master key, so occ backup:target:add opens
|
|
||||||
the existing repository on the location instead of creating a new, empty one.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
occ backup:key:import /root/ng_backup-recovery-kit.json
|
|
||||||
occ backup:target:add offsite sftp -a password::password -o host=... -o user=... -o password=...
|
|
||||||
occ backup:restore:full offsite <snapshot>
|
|
||||||
HELP);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
if ($this->keys->isInitialized()) {
|
|
||||||
$output->writeln('<error>A backup key already exists here (fingerprint ' . $this->keys->fingerprint() . '). occ backup:key:import is only for a fresh installation.</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$kitFile = (string)$input->getArgument('kit-file');
|
|
||||||
$raw = @file_get_contents($kitFile);
|
|
||||||
if ($raw === false) {
|
|
||||||
$output->writeln("<error>Cannot read $kitFile</error>");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$kit = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
} catch (\JsonException $e) {
|
|
||||||
$output->writeln('<error>Not a valid recovery kit file: ' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$file = $input->getOption('passphrase-file');
|
|
||||||
if ($file !== null) {
|
|
||||||
$raw = @file_get_contents($file);
|
|
||||||
if ($raw === false) {
|
|
||||||
$output->writeln('<error>Cannot read the passphrase file</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$pass = rtrim($raw, "\r\n");
|
|
||||||
} else {
|
|
||||||
$helper = $this->getHelper('question');
|
|
||||||
$q = (new Question('Passphrase of one of the kit\'s slots: '))->setHidden(true)->setHiddenFallback(false);
|
|
||||||
$pass = (string)$helper->ask($input, $output, $q);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$this->keys->importRecoveryKit($kit, $pass);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$output->writeln('Backup key imported, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
|
||||||
// Read from the kit file, which is not authenticated beyond the wrapped key: show it so a
|
|
||||||
// wrong or altered value stands out.
|
|
||||||
$output->writeln('Deletion delay for forgotten snapshots: <info>' . $this->keys->deleteDelayDays() . '</info> days (from the kit).');
|
|
||||||
$output->writeln('Next: connect the backup location with the same details as before (<info>occ backup:target:add</info>), then <info>occ backup:list</info> and <info>occ backup:restore:full</info>.');
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,7 +14,7 @@ use Symfony\Component\Console\Input\InputOption;
|
|||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
use Symfony\Component\Console\Question\Question;
|
use Symfony\Component\Console\Question\Question;
|
||||||
|
|
||||||
final class KeyInit extends Command {
|
class KeyInit extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
) {
|
) {
|
||||||
@@ -25,8 +25,6 @@ final class KeyInit extends Command {
|
|||||||
$this->setName('backup:key:init')
|
$this->setName('backup:key:init')
|
||||||
->setDescription('Create the backup key, protected by a passphrase you keep safe')
|
->setDescription('Create the backup key, protected by a passphrase you keep safe')
|
||||||
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking')
|
->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file instead of asking')
|
||||||
->addOption('label', 'l', InputOption::VALUE_REQUIRED, 'Label of the first passphrase slot (who/where), e.g. "Safe"', 'Slot 1')
|
|
||||||
->addOption('delete-delay-days', null, InputOption::VALUE_REQUIRED, 'Days a removed snapshot stays in the trash before its data is deleted (1-365, fixed after this)', (string)KeyService::DEFAULT_DELETE_DELAY)
|
|
||||||
->setHelp(<<<'HELP'
|
->setHelp(<<<'HELP'
|
||||||
Creates the master key that encrypts all backups of this installation.
|
Creates the master key that encrypts all backups of this installation.
|
||||||
|
|
||||||
@@ -59,14 +57,12 @@ HELP);
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
$this->keys->initialize($pass, (int)$input->getOption('delete-delay-days'), (string)$input->getOption('label'));
|
$this->keys->initialize($pass);
|
||||||
} catch (\InvalidArgumentException $e) {
|
} catch (\InvalidArgumentException $e) {
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
$output->writeln('Backup key created, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
$output->writeln('Backup key created, fingerprint <info>' . $this->keys->fingerprint() . '</info>.');
|
||||||
$output->writeln('Removed snapshots stay in the trash on the location for <info>' . $this->keys->deleteDelayDays() . ' days</info> (cannot be changed later).');
|
|
||||||
$output->writeln('Tip: add up to two more passphrases for other people with occ backup:key:slot:add (e.g. CTO, head of IT).');
|
|
||||||
$output->writeln('Next: save the recovery kit with <info>occ backup:key:kit --output=FILE</info> and confirm with <info>occ backup:key:confirm</info>.');
|
$output->writeln('Next: save the recovery kit with <info>occ backup:key:kit --output=FILE</info> and confirm with <info>occ backup:key:confirm</info>.');
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-18
@@ -13,10 +13,9 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class KeyKit extends Command {
|
class KeyKit extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
private \OCA\NgBackup\Service\AlertService $alerts,
|
|
||||||
) {
|
) {
|
||||||
parent::__construct();
|
parent::__construct();
|
||||||
}
|
}
|
||||||
@@ -32,31 +31,17 @@ final class KeyKit extends Command {
|
|||||||
$output->writeln('<error>No backup key yet: run occ backup:key:init first.</error>');
|
$output->writeln('<error>No backup key yet: run occ backup:key:init first.</error>');
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
$this->alerts->securityEvent('kit_downloaded');
|
|
||||||
$json = json_encode($this->keys->recoveryKit(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n";
|
$json = json_encode($this->keys->recoveryKit(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n";
|
||||||
$file = $input->getOption('output');
|
$file = $input->getOption('output');
|
||||||
if ($file === null) {
|
if ($file === null) {
|
||||||
$output->write($json, false, OutputInterface::OUTPUT_RAW);
|
$output->write($json, false, OutputInterface::OUTPUT_RAW);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
// Created owner-only from the start (no window where it is world-readable), never over an
|
if (file_put_contents($file, $json) === false) {
|
||||||
// existing file or symlink someone else may have placed at a predictable path.
|
|
||||||
$old = umask(0077);
|
|
||||||
try {
|
|
||||||
$fh = @fopen($file, 'xb');
|
|
||||||
} finally {
|
|
||||||
umask($old);
|
|
||||||
}
|
|
||||||
if ($fh === false) {
|
|
||||||
$output->writeln("<error>Cannot create $file (it must not exist yet)</error>");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$ok = fwrite($fh, $json) === strlen($json);
|
|
||||||
if (!fclose($fh) || !$ok) {
|
|
||||||
@unlink($file);
|
|
||||||
$output->writeln("<error>Cannot write $file</error>");
|
$output->writeln("<error>Cannot write $file</error>");
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
@chmod($file, 0600);
|
||||||
$output->writeln("Recovery kit written to <info>$file</info>. Copy it off this server, then run <info>occ backup:key:confirm</info>.");
|
$output->writeln("Recovery kit written to <info>$file</info>. Copy it off this server, then run <info>occ backup:key:confirm</info>.");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,17 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\KeyService;
|
|
||||||
use OCA\NgBackup\Service\KeySlotService;
|
|
||||||
|
|
||||||
final class KeySlotAdd extends KeySlots {
|
|
||||||
public function __construct(KeyService $keys, KeySlotService $slots) {
|
|
||||||
parent::__construct($keys, $slots, 'add');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\KeyService;
|
|
||||||
use OCA\NgBackup\Service\KeySlotService;
|
|
||||||
|
|
||||||
final class KeySlotRemove extends KeySlots {
|
|
||||||
public function __construct(KeyService $keys, KeySlotService $slots) {
|
|
||||||
parent::__construct($keys, $slots, 'remove');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\KeyService;
|
|
||||||
use OCA\NgBackup\Service\KeySlotService;
|
|
||||||
|
|
||||||
final class KeySlotReplace extends KeySlots {
|
|
||||||
public function __construct(KeyService $keys, KeySlotService $slots) {
|
|
||||||
parent::__construct($keys, $slots, 'replace');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\KeyService;
|
|
||||||
use OCA\NgBackup\Service\KeySlotService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Helper\Table;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
use Symfony\Component\Console\Question\Question;
|
|
||||||
|
|
||||||
/** backup:key:slots, backup:key:slot:add, backup:key:slot:replace, backup:key:slot:remove */
|
|
||||||
class KeySlots extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private KeyService $keys,
|
|
||||||
private KeySlotService $slots,
|
|
||||||
private string $action = 'list',
|
|
||||||
) {
|
|
||||||
// list is the default; subclasses pick another action
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$help = <<<'HELP'
|
|
||||||
Up to three passphrases ("slots") can open the backup key, so that one person leaving or dying
|
|
||||||
does not lock you out. Example: one passphrase in the safe, one with the CTO, one with the head
|
|
||||||
of IT. Each holder keeps a copy of the same recovery kit and only knows their own passphrase.
|
|
||||||
After every change: occ backup:key:kit and occ backup:key:confirm.
|
|
||||||
HELP;
|
|
||||||
match ($this->action) {
|
|
||||||
'add' => $this->setName('backup:key:slot:add')->setDescription('Add a passphrase slot')
|
|
||||||
->addOption('label', 'l', InputOption::VALUE_REQUIRED, 'Who or where, e.g. "CTO" or "Safe"', 'Slot'),
|
|
||||||
'replace' => $this->setName('backup:key:slot:replace')->setDescription('Set a new passphrase for a slot (e.g. its holder left)')
|
|
||||||
->addArgument('slot', InputArgument::REQUIRED, 'Slot number (see backup:key:slots)')
|
|
||||||
->addOption('label', 'l', InputOption::VALUE_REQUIRED, 'New label'),
|
|
||||||
'remove' => $this->setName('backup:key:slot:remove')->setDescription('Remove a passphrase slot (not the last one)')
|
|
||||||
->addArgument('slot', InputArgument::REQUIRED, 'Slot number (see backup:key:slots)'),
|
|
||||||
default => $this->setName('backup:key:slots')->setDescription('List the passphrase slots of the backup key'),
|
|
||||||
};
|
|
||||||
if ($this->action === 'add' || $this->action === 'replace') {
|
|
||||||
$this->addOption('passphrase-file', null, InputOption::VALUE_REQUIRED, 'Read the passphrase from this file');
|
|
||||||
}
|
|
||||||
$this->setHelp($help);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
if (!$this->keys->isInitialized()) {
|
|
||||||
$output->writeln('<error>No backup key yet: run occ backup:key:init first.</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
switch ($this->action) {
|
|
||||||
case 'add':
|
|
||||||
$n = $this->slots->add($this->passphrase($input, $output), (string)$input->getOption('label'));
|
|
||||||
$output->writeln("Slot $n added.");
|
|
||||||
break;
|
|
||||||
case 'replace':
|
|
||||||
$this->slots->replace((int)$input->getArgument('slot'), $this->passphrase($input, $output), $input->getOption('label'));
|
|
||||||
$output->writeln('Slot ' . (int)$input->getArgument('slot') . ' has a new passphrase.');
|
|
||||||
break;
|
|
||||||
case 'remove':
|
|
||||||
$this->slots->remove((int)$input->getArgument('slot'));
|
|
||||||
$output->writeln('Slot ' . (int)$input->getArgument('slot') . ' removed.');
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
(new Table($output))->setHeaders(['slot', 'label', 'set on'])
|
|
||||||
->setRows(array_map(fn ($s) => [$s['slot'], $s['label'], $s['created'] ? date('Y-m-d H:i', $s['created']) : ''], $this->keys->slots()))->render();
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$output->writeln('Now download and confirm the new recovery kit: occ backup:key:kit -o FILE && occ backup:key:confirm');
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function passphrase(InputInterface $input, OutputInterface $output): string {
|
|
||||||
if ($file = $input->getOption('passphrase-file')) {
|
|
||||||
return rtrim((string)@file_get_contents($file), "\r\n");
|
|
||||||
}
|
|
||||||
$h = $this->getHelper('question');
|
|
||||||
$p = (string)$h->ask($input, $output, (new Question('Passphrase (min. 12 characters): '))->setHidden(true)->setHiddenFallback(false));
|
|
||||||
if ($p !== (string)$h->ask($input, $output, (new Question('Repeat: '))->setHidden(true)->setHiddenFallback(false))) {
|
|
||||||
throw new \InvalidArgumentException('The passphrases do not match');
|
|
||||||
}
|
|
||||||
return $p;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputArgument;
|
|||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class ListSnapshots extends Command {
|
class ListSnapshots extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private SnapshotMapper $snapshots,
|
private SnapshotMapper $snapshots,
|
||||||
|
|||||||
@@ -1,70 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\PruneService;
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class Prune extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private PruneService $prune,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:prune')
|
|
||||||
->setDescription('Apply the retention policy: remove old snapshots and the data only they used')
|
|
||||||
->addArgument('target', InputArgument::OPTIONAL, 'Location name or id (default: all that are not append-only)')
|
|
||||||
->addOption('dry-run', null, InputOption::VALUE_NONE, 'Only show what would be removed');
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$name = $input->getArgument('target');
|
|
||||||
$targets = $name === null ? array_filter($this->targets->list(), fn ($t) => !$t->getAppendOnly()) : [$this->targets->get($name)];
|
|
||||||
$dry = (bool)$input->getOption('dry-run');
|
|
||||||
$output->writeln('Policy: ' . json_encode($this->prune->policy()->toArray()) . ($dry ? ' (dry run)' : ''));
|
|
||||||
$rc = 0;
|
|
||||||
foreach ($targets as $t) {
|
|
||||||
// User exports first: Repository::prune()'s blob garbage collection (run as part of
|
|
||||||
// apply() below) scans the users/ manifests still on the location at that point, so
|
|
||||||
// forgetting one here lets its now-unused blobs be freed in this same pass.
|
|
||||||
try {
|
|
||||||
$ru = $this->prune->applyUserExports($t, $dry);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
// Snapshot retention is independent of this: report and carry on with it.
|
|
||||||
$output->writeln('<error>' . $t->getName() . ' (user exports): ' . $e->getMessage() . '</error>');
|
|
||||||
$rc = 1;
|
|
||||||
$ru = ['forgotten' => []];
|
|
||||||
}
|
|
||||||
if ($ru['forgotten'] !== []) {
|
|
||||||
$affected = count(array_unique(array_map(static fn (string $path): string => explode('/', $path)[1] ?? '', $ru['forgotten'])));
|
|
||||||
$output->writeln(sprintf('<info>%s</info>: %s %d user export(s) for %d user(s)',
|
|
||||||
$t->getName(), $dry ? 'would forget' : 'forgot', count($ru['forgotten']), $affected));
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$r = $this->prune->apply($t, $dry);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $t->getName() . ': ' . $e->getMessage() . '</error>');
|
|
||||||
$rc = 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$p = $r['prune'];
|
|
||||||
$output->writeln(sprintf('<info>%s</info>: keep %d, %s %d snapshots; packs: %d, %s %d, repack %d, frees %.1f MiB',
|
|
||||||
$t->getName(), count($r['kept']), $dry ? 'would forget' : 'forgot', count($r['forgotten']), $p['packs'],
|
|
||||||
$dry ? 'would delete' : 'deleted', $p['deleted'], $p['repacked'], $p['freedBytes'] / 1048576));
|
|
||||||
}
|
|
||||||
return $rc;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class Restore extends Command {
|
class Restore extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RestoreService $restore,
|
private RestoreService $restore,
|
||||||
@@ -49,7 +49,7 @@ HELP);
|
|||||||
try {
|
try {
|
||||||
$target = $this->targets->get($input->getArgument('target'));
|
$target = $this->targets->get($input->getArgument('target'));
|
||||||
$snapshot = $input->getArgument('snapshot');
|
$snapshot = $input->getArgument('snapshot');
|
||||||
$path = trim((string)$input->getArgument('path'), '/'); // CLI convenience; the service itself only accepts canonical paths
|
$path = $input->getArgument('path');
|
||||||
if ($dir = $input->getOption('to-directory')) {
|
if ($dir = $input->getOption('to-directory')) {
|
||||||
$n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir);
|
$n = $this->restore->restoreToDirectory($target, $snapshot, $path, $dir);
|
||||||
$output->writeln("Restored <info>$n</info> files to $dir.");
|
$output->writeln("Restored <info>$n</info> files to $dir.");
|
||||||
|
|||||||
@@ -1,97 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\RestoreFullService;
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use OCP\IUserManager;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Helper\QuestionHelper;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class RestoreFull extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private RestoreFullService $restoreFull,
|
|
||||||
private IUserManager $userManager,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:restore:full')
|
|
||||||
->setDescription('Disaster recovery: restore the database, data directory and config on a fresh installation')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
|
||||||
->addOption('force', null, InputOption::VALUE_NONE, 'Also on an installation that already has more than one user, and without asking')
|
|
||||||
->setHelp(<<<'HELP'
|
|
||||||
For a FRESH Nextcloud installation only, same major version as the backup, right after
|
|
||||||
occ maintenance:install, occ backup:key:import and occ backup:target:add -- before anyone else
|
|
||||||
uses this instance.
|
|
||||||
|
|
||||||
Every app that was enabled when the snapshot was taken must already be installed here (its
|
|
||||||
database tables must exist before the restore can write into them); the command lists what is
|
|
||||||
missing and refuses to continue rather than guess.
|
|
||||||
|
|
||||||
Steps: the database is replaced (consistent, in one transaction); the data directory is restored
|
|
||||||
directly in place; config.php is merged in -- everything from the backup except the database
|
|
||||||
connection and data directory settings, which stay as occ maintenance:install configured them
|
|
||||||
here. Other files under config/ (if any) are not restored.
|
|
||||||
|
|
||||||
Afterwards: check trusted_domains and overwritehost/overwriteprotocol match this server, and
|
|
||||||
occ maintenance:repair.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
occ backup:restore:full offsite 537bf7e9c1a4...
|
|
||||||
HELP);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$force = (bool)$input->getOption('force');
|
|
||||||
// A whole-server restore replaces the database and data directory of THIS instance. On
|
|
||||||
// anything but a fresh install (only the admin from occ maintenance:install) that is
|
|
||||||
// almost certainly a mistake, so it needs --force.
|
|
||||||
$users = $this->userManager->countUsersTotal(2);
|
|
||||||
if (!$force && ($users === false || $users > 1)) {
|
|
||||||
$output->writeln('<error>This installation already has more than one user: backup:restore:full would replace its whole database and data directory. It is meant for a fresh installation. Use --force if this really is what you want.</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
if (!$force && $input->isInteractive()) {
|
|
||||||
$q = new ConfirmationQuestion('This replaces the database, the data directory and config.php of this installation. Continue? [y/N] ', false);
|
|
||||||
if (!(new QuestionHelper())->ask($input, $output, $q)) {
|
|
||||||
$output->writeln('Cancelled.');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$target = $this->targets->get($input->getArgument('target'));
|
|
||||||
$snapshot = (string)$input->getArgument('snapshot');
|
|
||||||
$result = $this->restoreFull->restoreFull($target, $snapshot);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
$output->writeln('If this happened after the restore had started, the instance may be partly restored: fix the cause and run the same command again with --force.');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$output->writeln(sprintf(
|
|
||||||
'Database restored (<info>%d</info> tables, <info>%d</info> unchanged), config merged (<info>%d</info> keys), <info>%d</info> data files restored.',
|
|
||||||
count($result['dbReport']['restored']),
|
|
||||||
count($result['dbReport']['skipped']),
|
|
||||||
$result['configKeys'],
|
|
||||||
$result['dataFiles'],
|
|
||||||
));
|
|
||||||
if (!$result['dbConsistent']) {
|
|
||||||
$output->writeln('<comment>Warning: this snapshot\'s database dump was made over several steps (web cron), so it is not one consistent point in time. Check the instance carefully; prefer a snapshot made with system cron.</comment>');
|
|
||||||
}
|
|
||||||
$output->writeln('Next: check trusted_domains and overwritehost/overwriteprotocol match this server, then occ maintenance:repair.');
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use OCA\NgBackup\Service\UserRestoreService;
|
|
||||||
use OCA\NgBackup\UserMigration\UserMigrationInstaller;
|
|
||||||
use OCP\IUserManager;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Helper\QuestionHelper;
|
|
||||||
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class RestoreUser extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private UserRestoreService $userRestore,
|
|
||||||
private UserMigrationInstaller $installer,
|
|
||||||
private IUserManager $userManager,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:user:restore')
|
|
||||||
->setDescription('Restore a user from a user_migration export (see backup:user:backup)')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->addArgument('manifest', InputArgument::REQUIRED, 'Manifest path printed by backup:user:backup (users/<uid>/<exportId>)')
|
|
||||||
->addOption('mode', 'm', InputOption::VALUE_REQUIRED, 'replace or as-backup', UserRestoreService::MODE_REPLACE)
|
|
||||||
->addOption('force', null, InputOption::VALUE_NONE, 'Replace an existing account without asking (required when not interactive)')
|
|
||||||
->addOption('install-user-migration', null, InputOption::VALUE_NONE, 'Install and enable user_migration if it is missing (non-interactive consent)')
|
|
||||||
->setHelp(<<<'HELP'
|
|
||||||
Never imports over a live account. Modes:
|
|
||||||
replace (default) safety-exports the current account first, then deletes and recreates it
|
|
||||||
fresh from the backup, under the same user id
|
|
||||||
as-backup restores into a new account "<uid>-bak" (or "-bak-2", ...) instead, so an admin can
|
|
||||||
compare and transfer by hand; Nextcloud cannot rename a user id, so the backup gets
|
|
||||||
the new name, never the existing account
|
|
||||||
|
|
||||||
Example:
|
|
||||||
occ backup:user:restore offsite users/alice/20261005T120000Z-ab12cd34
|
|
||||||
occ backup:user:restore offsite users/alice/20261005T120000Z-ab12cd34 --mode=as-backup
|
|
||||||
HELP);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$mode = (string)$input->getOption('mode');
|
|
||||||
$uid = explode('/', (string)$input->getArgument('manifest'))[1] ?? '';
|
|
||||||
if ($mode === UserRestoreService::MODE_REPLACE && $uid !== '' && $this->userManager->userExists($uid) && !$input->getOption('force')) {
|
|
||||||
// Replace deletes the live account (after a safety export). Shares to others and data of
|
|
||||||
// apps without a user_migration migrator do not come back, so never do it unasked.
|
|
||||||
if (!$input->isInteractive()) {
|
|
||||||
$output->writeln("<error>$uid exists: --mode=replace deletes and recreates it. Add --force, or use --mode=as-backup.</error>");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$q = new ConfirmationQuestion("$uid exists and will be deleted and recreated from the backup (a safety export is made first; its shares to others and data of apps without a migrator are not restored). Continue? [y/N] ", false);
|
|
||||||
if (!(new QuestionHelper())->ask($input, $output, $q)) {
|
|
||||||
$output->writeln('Cancelled.');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$output->writeln('<comment>Note: on restore, user_migration only imports the app settings on its own allowlist (e.g. calendar view and reminder settings); other app settings of this user are skipped. Reason: NG Backup uses user_migration\'s importer; widening the list would mean building a settings importer of our own.</comment>');
|
|
||||||
$target = $this->targets->get($input->getArgument('target'));
|
|
||||||
$this->installer->ensureEnabled($input, $output);
|
|
||||||
$user = $this->userRestore->restoreUser($target, (string)$input->getArgument('manifest'), $mode);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$output->writeln('Restored <info>' . $user->getUID() . '</info>.');
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\PruneService;
|
|
||||||
use OCA\NgBackup\Service\RetentionPolicy;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class Retention extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private PruneService $prune,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:retention')
|
|
||||||
->setDescription('Show or set how many snapshots are kept')
|
|
||||||
->addOption('last', null, InputOption::VALUE_REQUIRED, 'Always keep the newest N')
|
|
||||||
->addOption('daily', null, InputOption::VALUE_REQUIRED, 'Keep one per day for the last N days with a backup')
|
|
||||||
->addOption('weekly', null, InputOption::VALUE_REQUIRED, 'Keep one per week for N weeks')
|
|
||||||
->addOption('monthly', null, InputOption::VALUE_REQUIRED, 'Keep one per month for N months')
|
|
||||||
->addOption('user-last', null, InputOption::VALUE_REQUIRED, 'Keep the newest N of each user\'s own exports (occ backup:user:backup)')
|
|
||||||
->setHelp("Example: occ backup:retention --last=3 --daily=7 --weekly=4 --monthly=12 --user-last=3");
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
if ($input->getOption('user-last') !== null && !ctype_digit((string)$input->getOption('user-last'))) {
|
|
||||||
$output->writeln('<error>--user-last must be a whole number</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$p = $this->prune->policy()->toArray();
|
|
||||||
$changed = false;
|
|
||||||
foreach (['last', 'daily', 'weekly', 'monthly'] as $k) {
|
|
||||||
if ($input->getOption($k) !== null) {
|
|
||||||
$p[$k] = max(0, (int)$input->getOption($k));
|
|
||||||
$changed = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($changed) {
|
|
||||||
try {
|
|
||||||
$this->prune->setPolicy(RetentionPolicy::fromArray($p));
|
|
||||||
} catch (\InvalidArgumentException $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($input->getOption('user-last') !== null) {
|
|
||||||
try {
|
|
||||||
$this->prune->setUserExportRetention((int)$input->getOption('user-last'));
|
|
||||||
} catch (\InvalidArgumentException $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$output->writeln(sprintf('Keep the last %d, plus one per day for %d days, per week for %d weeks, per month for %d months.', $p['last'], $p['daily'], $p['weekly'], $p['monthly']));
|
|
||||||
$output->writeln(sprintf('Keep the newest %d of each user\'s own exports.', $this->prune->userExportRetention()));
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -16,7 +16,7 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class RunBackup extends Command {
|
class RunBackup extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private BackupService $backups,
|
private BackupService $backups,
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
@@ -75,13 +75,6 @@ HELP);
|
|||||||
});
|
});
|
||||||
if ($run->getStatus() === Run::DONE) {
|
if ($run->getStatus() === Run::DONE) {
|
||||||
$output->writeln(sprintf(' done in %.0fs', microtime(true) - $start));
|
$output->writeln(sprintf(' done in %.0fs', microtime(true) - $start));
|
||||||
$f = json_decode($run->getStats() ?? '{}', true)['files'] ?? [];
|
|
||||||
if (($f['unreadable'] ?? 0) > 0) {
|
|
||||||
$output->writeln(sprintf('<comment> warning: %d unreadable files or folders are missing from this backup (see the server log)</comment>', $f['unreadable']));
|
|
||||||
}
|
|
||||||
if (($f['changing'] ?? 0) > 0) {
|
|
||||||
$output->writeln(sprintf('<comment> warning: %d files kept changing and were saved as read (see the server log)</comment>', $f['changing']));
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
$output->writeln('<error> ' . $run->getStatus() . ': ' . $run->getError() . '</error>');
|
$output->writeln('<error> ' . $run->getStatus() . ': ' . $run->getError() . '</error>');
|
||||||
$failed++;
|
$failed++;
|
||||||
|
|||||||
+4
-28
@@ -10,18 +10,16 @@ namespace OCA\NgBackup\Command;
|
|||||||
use OCA\NgBackup\Db\RunMapper;
|
use OCA\NgBackup\Db\RunMapper;
|
||||||
use OCA\NgBackup\Service\KeyService;
|
use OCA\NgBackup\Service\KeyService;
|
||||||
use OCA\NgBackup\Service\TargetService;
|
use OCA\NgBackup\Service\TargetService;
|
||||||
use OCA\NgBackup\Service\VerifyService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
use Symfony\Component\Console\Command\Command;
|
||||||
use Symfony\Component\Console\Helper\Table;
|
use Symfony\Component\Console\Helper\Table;
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class Status extends Command {
|
class Status extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private RunMapper $runs,
|
private RunMapper $runs,
|
||||||
private VerifyService $verify,
|
|
||||||
) {
|
) {
|
||||||
parent::__construct();
|
parent::__construct();
|
||||||
}
|
}
|
||||||
@@ -46,35 +44,13 @@ final class Status extends Command {
|
|||||||
$rows = [];
|
$rows = [];
|
||||||
foreach ($this->runs->findRecent(10) as $r) {
|
foreach ($this->runs->findRecent(10) as $r) {
|
||||||
$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
|
$s = json_decode($r->getStats() ?? '{}', true)['files'] ?? [];
|
||||||
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . self::percent($r) . ')' : ''),
|
$rows[] = [$r->getId(), $names[$r->getTargetId()] ?? '#' . $r->getTargetId(), $r->getKind(), $r->getStatus() . ($r->getStatus() === 'running' ? ' (' . $r->getPhase() . ')' : ''),
|
||||||
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? gmdate('H:i:s', $r->getFinishedAt() - $r->getStartedAt()) : '',
|
date('Y-m-d H:i', $r->getStartedAt()), $r->getFinishedAt() ? gmdate('H:i:s', $r->getFinishedAt() - $r->getStartedAt()) : '',
|
||||||
$s['files'] ?? '', isset($s['bytes']) ? self::size((int)$s['bytes']) : '',
|
$s['files'] ?? '', isset($s['uploaded']) ? sprintf('%.1f MiB', $s['uploaded'] / 1048576) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
|
||||||
isset($s['uploaded']) ? self::size((int)$s['uploaded']) : '', mb_strimwidth((string)$r->getError(), 0, 50, '…')];
|
|
||||||
}
|
}
|
||||||
if ($rows !== []) {
|
if ($rows !== []) {
|
||||||
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'size', 'uploaded', 'error'])->setRows($rows)->render();
|
(new Table($output))->setHeaders(['run', 'location', 'kind', 'status', 'started', 'took', 'files', 'uploaded', 'error'])->setRows($rows)->render();
|
||||||
}
|
|
||||||
foreach ($this->verify->progress() as $targetId => $state) {
|
|
||||||
$output->writeln(sprintf('Verify of %s on %s: %s', mb_strimwidth((string)($state['what'] ?? ''), 0, 40, '…'),
|
|
||||||
$names[$targetId] ?? '#' . $targetId, VerifyService::describe($state)));
|
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** " 68%", or " ~68%" when estimated from the file cache; empty without an estimate */
|
|
||||||
private static function size(int $bytes): string {
|
|
||||||
$b = (float)$bytes;
|
|
||||||
foreach (['B', 'KiB', 'MiB', 'GiB'] as $u) {
|
|
||||||
if ($b < 1024) {
|
|
||||||
return $u === 'B' ? "$bytes B" : sprintf('%.1f %s', $b, $u);
|
|
||||||
}
|
|
||||||
$b /= 1024.0;
|
|
||||||
}
|
|
||||||
return sprintf('%.1f TiB', $b);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function percent(\OCA\NgBackup\Db\Run $r): string {
|
|
||||||
$p = \OCA\NgBackup\Job\BackupRun::progressOf($r);
|
|
||||||
return $p === null ? '' : sprintf(' %s%d%%', $p['approximate'] ? '~' : '', (int)floor($p['fraction'] * 100.0));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class TargetAdd extends Command {
|
class TargetAdd extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
|
|||||||
@@ -1,45 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class TargetAppendOnly extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private \OCA\NgBackup\Service\AlertService $alerts,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:target:append-only')
|
|
||||||
->setDescription('Make a location append-only (NG Backup never deletes there) or normal again')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->addArgument('state', InputArgument::REQUIRED, 'on or off')
|
|
||||||
->setHelp("Append-only protects backups if this server is compromised: combine it with credentials\nwithout delete rights and a lifecycle rule or object lock on the location itself.");
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$state = strtolower((string)$input->getArgument('state'));
|
|
||||||
if (!in_array($state, ['on', 'off'], true)) {
|
|
||||||
$output->writeln('<error>state must be on or off</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$t = $this->targets->setAppendOnly($this->targets->get($input->getArgument('target')), $state === 'on');
|
|
||||||
if ($state === 'off') {
|
|
||||||
$this->alerts->securityEvent('append_only_off', ['target' => $t->getName()]);
|
|
||||||
}
|
|
||||||
$output->writeln($t->getName() . ' is ' . ($t->getAppendOnly() ? 'append-only' : 'normal') . '.');
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -12,7 +12,7 @@ use OCA\NgBackup\Service\TargetService;
|
|||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class TargetBackends extends Base {
|
class TargetBackends extends Base {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -1,71 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\AlertService;
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class TargetEdit extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private AlertService $alerts,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:target:edit')
|
|
||||||
->setDescription('Change the connection settings or the name of a backup location, keeping its backups')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->addOption('name', null, InputOption::VALUE_REQUIRED, 'New name')
|
|
||||||
->addOption('backend', 'b', InputOption::VALUE_REQUIRED, 'New files_external backend id (see backup:target:backends)')
|
|
||||||
->addOption('auth', 'a', InputOption::VALUE_REQUIRED, 'New auth mechanism id')
|
|
||||||
->addOption('option', 'o', InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Option key=value to change (repeatable); other options keep their value')
|
|
||||||
->addOption('path', 'p', InputOption::VALUE_REQUIRED, 'New folder on the location')
|
|
||||||
->setHelp(<<<'HELP'
|
|
||||||
The new settings must lead to the same repository as before, so the history of the location is
|
|
||||||
kept. Use it after a password change, a new host name, or after moving the repository.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
occ backup:target:edit nas -o password=new-secret
|
|
||||||
HELP);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
try {
|
|
||||||
$t = $this->targets->get($input->getArgument('target'));
|
|
||||||
$options = $this->targets->options($t);
|
|
||||||
if ($input->getOption('backend') !== null || $input->getOption('auth') !== null) {
|
|
||||||
$options = []; // a different connection type has its own options
|
|
||||||
}
|
|
||||||
foreach ($input->getOption('option') as $pair) {
|
|
||||||
if (!str_contains($pair, '=')) {
|
|
||||||
$output->writeln("<error>Option '$pair' must be key=value</error>");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$k = (string)strstr($pair, '=', true);
|
|
||||||
$v = substr($pair, strlen($k) + 1);
|
|
||||||
$options[$k] = match (strtolower($v)) { 'true' => true, 'false' => false, default => $v };
|
|
||||||
}
|
|
||||||
$before = $t->getName();
|
|
||||||
$t = $this->targets->update($t, $input->getOption('name') ?? $t->getName(), $input->getOption('backend') ?? $t->getBackend(),
|
|
||||||
$input->getOption('auth') ?? $t->getAuth(), $options, $input->getOption('path') ?? $t->getBasePath());
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$this->alerts->securityEvent('target_changed', ['target' => $before]);
|
|
||||||
$output->writeln(sprintf('Location <info>%s</info> updated; repository %s found with the new settings.', $t->getName(), (string)$t->getRepositoryId()));
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class TargetLimit extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:target:limit')
|
|
||||||
->setDescription('Set a maximum size for the backups stored on a location (e.g. 100G), or "none"')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->addArgument('size', InputArgument::OPTIONAL, 'Size like 500M, 20G, 2T, or "none"; omit to show the current limit')
|
|
||||||
->setHelp("A backup that would exceed the limit stops with an error and administrators are notified.\n"
|
|
||||||
. "Note: with versioning or Object Lock on the location, removed data keeps using space there\n"
|
|
||||||
. "until it expires; keep the limit below what you are willing to pay for.");
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$t = $this->targets->get($input->getArgument('target'));
|
|
||||||
$size = $input->getArgument('size');
|
|
||||||
if ($size !== null) {
|
|
||||||
if (strtolower($size) === 'none') {
|
|
||||||
$t = $this->targets->setMaxBytes($t, null);
|
|
||||||
} elseif (preg_match('/^(\d+(?:\.\d+)?)\s*([KMGT]?)i?B?$/i', $size, $m)) {
|
|
||||||
$factor = ['' => 1, 'K' => 1024, 'M' => 1024 ** 2, 'G' => 1024 ** 3, 'T' => 1024 ** 4][strtoupper($m[2])];
|
|
||||||
$t = $this->targets->setMaxBytes($t, (int)round((float)$m[1] * $factor));
|
|
||||||
} else {
|
|
||||||
$output->writeln('<error>Use a size like 500M, 20G or 2T, or "none".</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$used = $this->targets->repository($t)->storedBytes();
|
|
||||||
$output->writeln(sprintf('%s: %.2f GiB stored, limit %s', $t->getName(), $used / 1073741824,
|
|
||||||
$t->getMaxBytes() === null ? 'none' : sprintf('%.2f GiB', $t->getMaxBytes() / 1073741824)));
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -13,7 +13,7 @@ use Symfony\Component\Console\Input\InputInterface;
|
|||||||
use Symfony\Component\Console\Input\InputOption;
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class TargetList extends Base {
|
class TargetList extends Base {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -13,10 +13,9 @@ use Symfony\Component\Console\Input\InputArgument;
|
|||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
final class TargetRemove extends Command {
|
class TargetRemove extends Command {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private \OCA\NgBackup\Service\AlertService $alerts,
|
|
||||||
) {
|
) {
|
||||||
parent::__construct();
|
parent::__construct();
|
||||||
}
|
}
|
||||||
@@ -35,7 +34,6 @@ final class TargetRemove extends Command {
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
$this->targets->remove($t);
|
$this->targets->remove($t);
|
||||||
$this->alerts->securityEvent('target_removed', ['target' => $t->getName()]);
|
|
||||||
$output->writeln('Removed <info>' . $t->getName() . '</info>. The backups on the location itself were not touched; add it again to use them.');
|
$output->writeln('Removed <info>' . $t->getName() . '</info>. The backups on the location itself were not touched; add it again to use them.');
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Db\AppConfigCatalogAnchor;
|
|
||||||
use OCA\NgBackup\Service\AlertService;
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use OCP\IAppConfig;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
use Symfony\Component\Console\Question\Question;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Accept the state a location shows now, after NG Backup refused it as rolled back. Only for
|
|
||||||
* legitimate cases (e.g. the location was restored from the provider's own snapshot), only from
|
|
||||||
* occ, audited and announced to all administrators.
|
|
||||||
*/
|
|
||||||
final class TargetTrustCurrent extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private IAppConfig $appConfig,
|
|
||||||
private AlertService $alerts,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:target:trust-current')
|
|
||||||
->setDescription('Accept the current state of a location that was refused as rolled back')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->setHelp("NG Backup refuses a location whose backup catalog is older than what this server saw\n"
|
|
||||||
. "before, because an attacker could have restored an old state. If you restored the location\n"
|
|
||||||
. "yourself (e.g. from the provider's snapshot), accept it with this command.");
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$t = $this->targets->get($input->getArgument('target'));
|
|
||||||
$output->writeln('<comment>Only continue if you know why the location shows an older state.</comment>');
|
|
||||||
$answer = (string)$this->getHelper('question')->ask($input, $output, new Question('Type the location name to accept its current state: '));
|
|
||||||
if ($answer !== $t->getName()) {
|
|
||||||
$output->writeln('<error>Not confirmed.</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$this->appConfig->deleteKey('ng_backup', 'catalog_' . preg_replace('/[^a-f0-9]/', '', (string)$t->getRepositoryId()));
|
|
||||||
$repo = $this->targets->repository($t); // re-anchors on the newest valid catalog
|
|
||||||
$this->alerts->securityEvent('catalog_trusted', ['target' => $t->getName(), 'generation' => (string)$repo->catalogGeneration()]);
|
|
||||||
$output->writeln(sprintf('Accepted generation %d of %s (%d snapshots).', $repo->catalogGeneration(), $t->getName(), count($repo->listSnapshots())));
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\KeyService;
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Helper\Table;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class Trash extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private KeyService $keys,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:trash')
|
|
||||||
->setDescription('List removed snapshots that can still be restored (backup:trash:restore)')
|
|
||||||
->addArgument('target', InputArgument::OPTIONAL, 'Location name or id (default: all)');
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
$name = $input->getArgument('target');
|
|
||||||
$delay = $this->keys->deleteDelayDays() * 86400;
|
|
||||||
$rows = [];
|
|
||||||
foreach ($name === null ? $this->targets->list() : [$this->targets->get($name)] as $t) {
|
|
||||||
foreach ($this->targets->repository($t)->trash() as $id => $info) {
|
|
||||||
$rows[] = [$t->getName(), $id, (string)($info['time'] ?? ''), $info['label'], date('Y-m-d H:i', $info['forgottenAt']), $info['by'],
|
|
||||||
date('Y-m-d H:i', $info['forgottenAt'] + $delay)];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($rows === []) {
|
|
||||||
$output->writeln('The trash is empty.');
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
(new Table($output))->setHeaders(['location', 'snapshot', 'made', 'label', 'removed', 'by', 'deleted after'])->setRows($rows)->render();
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\PruneService;
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class TrashRestore extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private PruneService $prune,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:trash:restore')
|
|
||||||
->setDescription('Bring a removed snapshot back from the trash')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:trash)');
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
try {
|
|
||||||
$this->prune->untrash($this->targets->get($input->getArgument('target')), $input->getArgument('snapshot'));
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$output->writeln('Snapshot restored from the trash.');
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Command;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use OCA\NgBackup\Service\VerifyService;
|
|
||||||
use Symfony\Component\Console\Command\Command;
|
|
||||||
use Symfony\Component\Console\Input\InputArgument;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Input\InputOption;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
|
|
||||||
final class Verify extends Command {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private VerifyService $verify,
|
|
||||||
) {
|
|
||||||
parent::__construct();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function configure(): void {
|
|
||||||
$this->setName('backup:verify')
|
|
||||||
->setDescription('Checksum audit of a snapshot: every blob it needs must be in the (freshly re-checked) index and its pack present')
|
|
||||||
->addArgument('target', InputArgument::REQUIRED, 'Location name or id')
|
|
||||||
->addArgument('snapshot', InputArgument::REQUIRED, 'Snapshot id (see backup:list)')
|
|
||||||
->addOption('deep', null, InputOption::VALUE_NONE, 'Also download and decrypt every blob (verifies its authentication tag and content hash, not just that it is listed); much slower')
|
|
||||||
->setHelp(<<<'HELP'
|
|
||||||
Without --deep this re-downloads and re-authenticates the index (never trusting the local cache)
|
|
||||||
and confirms every blob the snapshot needs is listed and its pack is present -- fast, no blob
|
|
||||||
content is read, so it cannot by itself catch a truncated or bit-rotted pack. With --deep every
|
|
||||||
blob is also downloaded and decrypted, which fails loudly on exactly that even if the index still
|
|
||||||
lists it -- the same verification an actual restore would do, run here deliberately instead of
|
|
||||||
waiting to find out during one.
|
|
||||||
|
|
||||||
While it runs it prints how far it is every 30 seconds; occ backup:status shows the same, and
|
|
||||||
how the last check of each location ended.
|
|
||||||
|
|
||||||
Run this periodically (cron, picking a sample of recent snapshots) rather than only right before
|
|
||||||
you'd need to rely on a restore.
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
occ backup:verify offsite 537bf7e9c1a4...
|
|
||||||
occ backup:verify offsite 537bf7e9c1a4... --deep
|
|
||||||
HELP);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function execute(InputInterface $input, OutputInterface $output): int {
|
|
||||||
try {
|
|
||||||
$target = $this->targets->get($input->getArgument('target'));
|
|
||||||
$ref = (string)$input->getArgument('snapshot');
|
|
||||||
$deep = (bool)$input->getOption('deep');
|
|
||||||
$onProgress = static function (array $state) use ($output): void {
|
|
||||||
$output->writeln(date('H:i:s') . ' ' . VerifyService::describe($state));
|
|
||||||
};
|
|
||||||
$result = str_starts_with($ref, 'users/')
|
|
||||||
? $this->verify->verifyUserExport($target, $ref, $deep, $onProgress)
|
|
||||||
: $this->verify->verify($target, $ref, $deep, $onProgress);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$output->writeln('<error>' . $e->getMessage() . '</error>');
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
$output->writeln(sprintf('%d files, %d blobs checked%s.', $result['filesChecked'], $result['blobsChecked'],
|
|
||||||
$deep ? sprintf(' (%.1f MiB decrypted)', $result['bytesChecked'] / 1048576) : ''));
|
|
||||||
if ($result['missing'] !== []) {
|
|
||||||
$output->writeln('<error>' . count($result['missing']) . ' blob(s) missing from the index:</error> ' . implode(', ', $result['missing']));
|
|
||||||
}
|
|
||||||
if ($result['failed'] !== []) {
|
|
||||||
$output->writeln('<error>' . count($result['failed']) . ' blob(s) failed verification (corrupted or tampered):</error> ' . implode(', ', $result['failed']));
|
|
||||||
}
|
|
||||||
if ($result['unreadable'] !== []) {
|
|
||||||
$output->writeln('<error>' . count($result['unreadable']) . ' blob(s) could not be read from the location</error> (often a temporary storage '
|
|
||||||
. 'or network error: run the check again; if the same blobs stay unreadable, their pack may be damaged or truncated): '
|
|
||||||
. implode(', ', $result['unreadable']));
|
|
||||||
$output->writeln('Last read error: ' . $result['readError']);
|
|
||||||
}
|
|
||||||
if ($result['missing'] === [] && $result['failed'] === [] && $result['unreadable'] === []) {
|
|
||||||
$output->writeln('<info>OK</info>');
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -19,8 +19,6 @@ use OCA\NgBackup\Service\RestoreService;
|
|||||||
use OCA\NgBackup\Service\TargetService;
|
use OCA\NgBackup\Service\TargetService;
|
||||||
use OCP\AppFramework\Controller;
|
use OCP\AppFramework\Controller;
|
||||||
use OCP\AppFramework\Http;
|
use OCP\AppFramework\Http;
|
||||||
use OCP\AppFramework\Http\Attribute\NoCSRFRequired;
|
|
||||||
use OCP\AppFramework\Http\Attribute\PasswordConfirmationRequired;
|
|
||||||
use OCP\AppFramework\Http\DataDownloadResponse;
|
use OCP\AppFramework\Http\DataDownloadResponse;
|
||||||
use OCP\AppFramework\Http\JSONResponse;
|
use OCP\AppFramework\Http\JSONResponse;
|
||||||
use OCP\BackgroundJob\IJobList;
|
use OCP\BackgroundJob\IJobList;
|
||||||
@@ -32,7 +30,7 @@ use OCP\IUserSession;
|
|||||||
* JSON API for the admin settings page. Every method requires an admin (the AppFramework
|
* JSON API for the admin settings page. Every method requires an admin (the AppFramework
|
||||||
* default for controllers without NoAdminRequired) and a valid CSRF token.
|
* default for controllers without NoAdminRequired) and a valid CSRF token.
|
||||||
*/
|
*/
|
||||||
final class AdminApiController extends Controller {
|
class AdminApiController extends Controller {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
IRequest $request,
|
IRequest $request,
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
@@ -44,9 +42,6 @@ final class AdminApiController extends Controller {
|
|||||||
private IAppConfig $appConfig,
|
private IAppConfig $appConfig,
|
||||||
private IJobList $jobs,
|
private IJobList $jobs,
|
||||||
private IUserSession $session,
|
private IUserSession $session,
|
||||||
private \OCA\NgBackup\Service\AlertService $alerts,
|
|
||||||
private \OCA\NgBackup\Service\KeySlotService $slots,
|
|
||||||
private \OCA\NgBackup\Service\PruneService $prune,
|
|
||||||
) {
|
) {
|
||||||
parent::__construct(Application::APP_ID, $request);
|
parent::__construct(Application::APP_ID, $request);
|
||||||
}
|
}
|
||||||
@@ -55,15 +50,12 @@ final class AdminApiController extends Controller {
|
|||||||
$targets = [];
|
$targets = [];
|
||||||
foreach ($this->targets->list() as $t) {
|
foreach ($this->targets->list() as $t) {
|
||||||
$targets[] = ['id' => $t->getId(), 'name' => $t->getName(), 'backend' => $t->getBackend(), 'path' => $t->getBasePath(),
|
$targets[] = ['id' => $t->getId(), 'name' => $t->getName(), 'backend' => $t->getBackend(), 'path' => $t->getBasePath(),
|
||||||
'repositoryId' => $t->getRepositoryId(), 'createdAt' => $t->getCreatedAt(), 'appendOnly' => (bool)$t->getAppendOnly(),
|
'repositoryId' => $t->getRepositoryId(), 'createdAt' => $t->getCreatedAt()];
|
||||||
'maxBytes' => $t->getMaxBytes(), 'storedBytes' => $this->alerts->lastStoredBytes($t)];
|
|
||||||
}
|
}
|
||||||
$runs = array_map(fn (Run $r) => $this->runJson($r), $this->runs->findRecent(15));
|
$runs = array_map(fn (Run $r) => $this->runJson($r), $this->runs->findRecent(15));
|
||||||
return new JSONResponse([
|
return new JSONResponse([
|
||||||
'key' => ['initialized' => $this->keys->isInitialized(), 'fingerprint' => $this->keys->fingerprint(),
|
'key' => ['initialized' => $this->keys->isInitialized(), 'fingerprint' => $this->keys->fingerprint(),
|
||||||
'confirmation' => $this->keys->recoveryKitConfirmation(), 'everConfirmed' => $this->keys->isInitialized() && $this->keys->everConfirmed(),
|
'confirmation' => $this->keys->recoveryKitConfirmation()],
|
||||||
'slots' => $this->keys->isInitialized() ? $this->keys->slots() : [], 'maxSlots' => \OCA\NgBackup\Service\KeyService::MAX_SLOTS,
|
|
||||||
'deleteDelayDays' => $this->keys->isInitialized() ? $this->keys->deleteDelayDays() : \OCA\NgBackup\Service\KeyService::DEFAULT_DELETE_DELAY],
|
|
||||||
'statement' => KeyConfirm::STATEMENT,
|
'statement' => KeyConfirm::STATEMENT,
|
||||||
'confirmationPhrase' => KeyConfirm::CONFIRMATION,
|
'confirmationPhrase' => KeyConfirm::CONFIRMATION,
|
||||||
'targets' => $targets,
|
'targets' => $targets,
|
||||||
@@ -72,149 +64,47 @@ final class AdminApiController extends Controller {
|
|||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[PasswordConfirmationRequired]
|
public function initKey(string $passphrase): JSONResponse {
|
||||||
public function initKey(string $passphrase, int $deleteDelayDays = \OCA\NgBackup\Service\KeyService::DEFAULT_DELETE_DELAY, string $label = 'Slot 1'): JSONResponse {
|
|
||||||
try {
|
try {
|
||||||
$this->keys->initialize($passphrase, $deleteDelayDays, $label);
|
$this->keys->initialize($passphrase);
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
}
|
}
|
||||||
return new JSONResponse(['fingerprint' => $this->keys->fingerprint()]);
|
return new JSONResponse(['fingerprint' => $this->keys->fingerprint()]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Plain download link (no CSRF token on a navigation), therefore guarded by a recent password
|
|
||||||
* confirmation instead, logged to the audit log and announced to all administrators. The kit
|
|
||||||
* only holds the passphrase-wrapped key: useless without the passphrase.
|
|
||||||
*/
|
|
||||||
#[NoCSRFRequired]
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function downloadKit(): DataDownloadResponse|JSONResponse {
|
public function downloadKit(): DataDownloadResponse|JSONResponse {
|
||||||
if (!$this->keys->isInitialized()) {
|
if (!$this->keys->isInitialized()) {
|
||||||
return new JSONResponse(['error' => 'No backup key yet'], Http::STATUS_BAD_REQUEST);
|
return new JSONResponse(['error' => 'No backup key yet'], Http::STATUS_BAD_REQUEST);
|
||||||
}
|
}
|
||||||
$this->alerts->securityEvent('kit_downloaded');
|
|
||||||
$json = json_encode($this->keys->recoveryKit(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
$json = json_encode($this->keys->recoveryKit(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
||||||
return new DataDownloadResponse($json, 'ng_backup-recovery-kit-' . $this->keys->fingerprint() . '.json', 'application/json');
|
return new DataDownloadResponse($json, 'ng_backup-recovery-kit-' . $this->keys->fingerprint() . '.json', 'application/json');
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The admin must tick the statement, send the exact phrase and the code from the kit file. */
|
/** The admin must tick the statement and send the exact confirmation phrase. */
|
||||||
#[PasswordConfirmationRequired]
|
public function confirmKit(bool $accepted, string $phrase): JSONResponse {
|
||||||
public function confirmKit(bool $accepted, string $phrase, string $code = ''): JSONResponse {
|
|
||||||
if (!$accepted || $phrase !== KeyConfirm::CONFIRMATION) {
|
if (!$accepted || $phrase !== KeyConfirm::CONFIRMATION) {
|
||||||
return new JSONResponse(['error' => 'Not confirmed'], Http::STATUS_BAD_REQUEST);
|
return new JSONResponse(['error' => 'Not confirmed'], Http::STATUS_BAD_REQUEST);
|
||||||
}
|
}
|
||||||
if (!$this->keys->checkKitCode($code)) {
|
|
||||||
return new JSONResponse(['error' => 'The confirmation code does not match the current recovery kit'], Http::STATUS_BAD_REQUEST);
|
|
||||||
}
|
|
||||||
$this->keys->confirmRecoveryKit($this->session->getUser()?->getUID() ?? 'unknown');
|
$this->keys->confirmRecoveryKit($this->session->getUser()?->getUID() ?? 'unknown');
|
||||||
return new JSONResponse(['confirmation' => $this->keys->recoveryKitConfirmation()]);
|
return new JSONResponse(['confirmation' => $this->keys->recoveryKitConfirmation()]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function addSlot(string $passphrase, string $label): JSONResponse {
|
|
||||||
try {
|
|
||||||
return new JSONResponse(['slot' => $this->slots->add($passphrase, $label)]);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function replaceSlot(int $slot, string $passphrase, ?string $label = null): JSONResponse {
|
|
||||||
try {
|
|
||||||
$this->slots->replace($slot, $passphrase, $label);
|
|
||||||
return new JSONResponse([]);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function removeSlot(int $slot): JSONResponse {
|
|
||||||
try {
|
|
||||||
$this->slots->remove($slot);
|
|
||||||
return new JSONResponse([]);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function trash(int $targetId): JSONResponse {
|
|
||||||
try {
|
|
||||||
$delay = $this->keys->deleteDelayDays() * 86400;
|
|
||||||
$out = [];
|
|
||||||
foreach ($this->targets->repository($this->targets->get((string)$targetId))->trash() as $id => $info) {
|
|
||||||
$out[] = ['id' => $id, 'label' => $info['label'], 'time' => $info['time'], 'removedAt' => $info['forgottenAt'],
|
|
||||||
'by' => $info['by'], 'deleteAfter' => $info['forgottenAt'] + $delay];
|
|
||||||
}
|
|
||||||
return new JSONResponse($out);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function untrash(int $targetId, string $snapshotId): JSONResponse {
|
|
||||||
try {
|
|
||||||
$this->prune->untrash($this->targets->get((string)$targetId), $snapshotId);
|
|
||||||
return new JSONResponse([]);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function backends(): JSONResponse {
|
public function backends(): JSONResponse {
|
||||||
return new JSONResponse($this->targets->availableBackends());
|
return new JSONResponse($this->targets->availableBackends());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[PasswordConfirmationRequired]
|
public function addTarget(string $name, string $backend, string $auth, array $options = [], string $path = 'ng_backup'): JSONResponse {
|
||||||
public function addTarget(string $name, string $backend, string $auth, array $options = [], string $path = 'ng_backup', ?float $maxGb = null): JSONResponse {
|
|
||||||
try {
|
try {
|
||||||
$r = $this->targets->add($name, $backend, $auth, $options, $path);
|
$r = $this->targets->add($name, $backend, $auth, $options, $path);
|
||||||
if ($maxGb !== null && $maxGb > 0) {
|
|
||||||
$this->targets->setMaxBytes($r['target'], (int)round($maxGb * 1073741824));
|
|
||||||
}
|
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
||||||
}
|
}
|
||||||
return new JSONResponse(['id' => $r['target']->getId(), 'created' => $r['created']]);
|
return new JSONResponse(['id' => $r['target']->getId(), 'created' => $r['created']]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The stored settings for the edit form; password values are left out (null). */
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function targetSettings(int $id): JSONResponse {
|
|
||||||
$t = $this->targets->get((string)$id);
|
|
||||||
return new JSONResponse(['name' => $t->getName(), 'backend' => $t->getBackend(), 'auth' => $t->getAuth(),
|
|
||||||
'options' => $this->targets->editableOptions($t), 'path' => $t->getBasePath()]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Options with a null value keep their stored value. */
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function updateTarget(int $id, string $name, string $backend, string $auth, array $options = [], string $path = 'ng_backup'): JSONResponse {
|
|
||||||
try {
|
|
||||||
$t = $this->targets->get((string)$id);
|
|
||||||
$before = $t->getName();
|
|
||||||
$t = $this->targets->update($t, $name, $backend, $auth, $options, $path);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
|
|
||||||
}
|
|
||||||
$this->alerts->securityEvent('target_changed', ['target' => $before]);
|
|
||||||
return new JSONResponse(['id' => $t->getId()]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Size limit in GiB; null or 0 = unlimited. */
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function setTargetLimit(int $id, ?float $maxGb = null): JSONResponse {
|
|
||||||
$t = $this->targets->setMaxBytes($this->targets->get((string)$id), ($maxGb !== null && $maxGb > 0) ? (int)round($maxGb * 1073741824) : null);
|
|
||||||
return new JSONResponse(['maxBytes' => $t->getMaxBytes()]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function removeTarget(int $id): JSONResponse {
|
public function removeTarget(int $id): JSONResponse {
|
||||||
$target = $this->targets->get((string)$id);
|
$this->targets->remove($this->targets->get((string)$id));
|
||||||
$this->targets->remove($target);
|
|
||||||
$this->alerts->securityEvent('target_removed', ['target' => $target->getName()]);
|
|
||||||
return new JSONResponse([]);
|
return new JSONResponse([]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -262,19 +152,13 @@ final class AdminApiController extends Controller {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Queue a restore of user files; it runs as a background job and shows up in the runs. */
|
/** Queue a restore of user files; it runs as a background job and shows up in the runs. */
|
||||||
#[PasswordConfirmationRequired]
|
|
||||||
public function startRestore(int $targetId, string $snapshotId, string $path, string $mode = RestoreService::MODE_NEW_FOLDER): JSONResponse {
|
public function startRestore(int $targetId, string $snapshotId, string $path, string $mode = RestoreService::MODE_NEW_FOLDER): JSONResponse {
|
||||||
if (!in_array($mode, [RestoreService::MODE_NEW_FOLDER, RestoreService::MODE_MERGE, RestoreService::MODE_REPLACE], true)
|
if (!in_array($mode, [RestoreService::MODE_NEW_FOLDER, RestoreService::MODE_MERGE, RestoreService::MODE_REPLACE], true)
|
||||||
|| !\OCA\NgBackup\Repository\Repository::isSafePath($path) || !preg_match('#^data/[^/]+/files(/.*)?$#', $path)) {
|
|| !preg_match('#^data/[^/]+/files(/.*)?$#', $path)) {
|
||||||
return new JSONResponse(['error' => 'Invalid path or mode'], Http::STATUS_BAD_REQUEST);
|
return new JSONResponse(['error' => 'Invalid path or mode'], Http::STATUS_BAD_REQUEST);
|
||||||
}
|
}
|
||||||
try {
|
|
||||||
$target = $this->targets->get((string)$targetId);
|
|
||||||
} catch (\Throwable) {
|
|
||||||
return new JSONResponse(['error' => 'No such location'], Http::STATUS_NOT_FOUND);
|
|
||||||
}
|
|
||||||
$run = new Run();
|
$run = new Run();
|
||||||
$run->setTargetId($target->getId());
|
$run->setTargetId($targetId);
|
||||||
$run->setKind('restore');
|
$run->setKind('restore');
|
||||||
$run->setStatus(Run::RUNNING);
|
$run->setStatus(Run::RUNNING);
|
||||||
$run->setPhase('queued');
|
$run->setPhase('queued');
|
||||||
@@ -283,28 +167,15 @@ final class AdminApiController extends Controller {
|
|||||||
$run->setSnapshotId($snapshotId);
|
$run->setSnapshotId($snapshotId);
|
||||||
$run->setStartedAt(time());
|
$run->setStartedAt(time());
|
||||||
$run->setUpdatedAt(time());
|
$run->setUpdatedAt(time());
|
||||||
try {
|
$run = $this->runs->insert($run);
|
||||||
$run = $this->prune->underStartLock($target, fn () => $this->runs->insert($run));
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
return new JSONResponse(['error' => $e->getMessage()], Http::STATUS_CONFLICT);
|
|
||||||
}
|
|
||||||
$this->jobs->add(RestoreJob::class, ['run' => $run->getId()]);
|
$this->jobs->add(RestoreJob::class, ['run' => $run->getId()]);
|
||||||
return new JSONResponse($this->runJson($run));
|
return new JSONResponse($this->runJson($run));
|
||||||
}
|
}
|
||||||
|
|
||||||
private function runJson(Run $r): array {
|
private function runJson(Run $r): array {
|
||||||
$snapshot = null;
|
|
||||||
if ($r->getSnapshotId() !== null) {
|
|
||||||
try {
|
|
||||||
$s = $this->snapshots->findOne($r->getTargetId(), $r->getSnapshotId());
|
|
||||||
$snapshot = ['createdAt' => $s->getCreatedAt(), 'label' => $s->getLabel()];
|
|
||||||
} catch (\Throwable) {
|
|
||||||
// Snapshot already pruned or trashed: the run keeps its id, just not the date.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ['id' => $r->getId(), 'targetId' => $r->getTargetId(), 'kind' => $r->getKind(), 'status' => $r->getStatus(),
|
return ['id' => $r->getId(), 'targetId' => $r->getTargetId(), 'kind' => $r->getKind(), 'status' => $r->getStatus(),
|
||||||
'phase' => $r->getPhase(), 'snapshotId' => $r->getSnapshotId(), 'snapshot' => $snapshot, 'startedAt' => $r->getStartedAt(),
|
'phase' => $r->getPhase(), 'snapshotId' => $r->getSnapshotId(), 'startedAt' => $r->getStartedAt(),
|
||||||
'updatedAt' => $r->getUpdatedAt(), 'finishedAt' => $r->getFinishedAt(), 'error' => $r->getError(),
|
'updatedAt' => $r->getUpdatedAt(), 'finishedAt' => $r->getFinishedAt(), 'error' => $r->getError(),
|
||||||
'stats' => json_decode($r->getStats() ?? 'null', true), 'progress' => \OCA\NgBackup\Job\BackupRun::progressOf($r)];
|
'stats' => json_decode($r->getStats() ?? 'null', true)];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,5 +7,5 @@ declare(strict_types=1);
|
|||||||
|
|
||||||
namespace OCA\NgBackup\Crypto;
|
namespace OCA\NgBackup\Crypto;
|
||||||
|
|
||||||
final class CryptoException extends \RuntimeException {
|
class CryptoException extends \RuntimeException {
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,32 +76,9 @@ final class KeyRing {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Open a key field that is either one wrapped key or {"slots": [{slot, label, key}, ...]}:
|
|
||||||
* every slot holds the same master key under its own passphrase.
|
|
||||||
*/
|
|
||||||
public static function unwrapAny(array $field, #[\SensitiveParameter] string $passphrase): self {
|
|
||||||
if (!isset($field['slots'])) {
|
|
||||||
return self::unwrap($field, $passphrase);
|
|
||||||
}
|
|
||||||
foreach ($field['slots'] as $slot) {
|
|
||||||
try {
|
|
||||||
return self::unwrap($slot['key'], $passphrase);
|
|
||||||
} catch (CryptoException) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new CryptoException('Wrong passphrase or damaged key');
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param array{v:int, salt:string, ops:int, mem:int, nonce:string, key:string} $wrapped */
|
/** @param array{v:int, salt:string, ops:int, mem:int, nonce:string, key:string} $wrapped */
|
||||||
public static function unwrap(array $wrapped, #[\SensitiveParameter] string $passphrase): self {
|
public static function unwrap(array $wrapped, #[\SensitiveParameter] string $passphrase): self {
|
||||||
$b64 = static fn (string $s): string => sodium_base642bin($s, SODIUM_BASE64_VARIANT_ORIGINAL);
|
$b64 = static fn (string $s): string => sodium_base642bin($s, SODIUM_BASE64_VARIANT_ORIGINAL);
|
||||||
// ops/mem come from unauthenticated JSON (kit file, repository config): bound them, so a
|
|
||||||
// damaged or crafted file cannot make the KDF use unbounded memory or time.
|
|
||||||
if ($wrapped['ops'] < SODIUM_CRYPTO_PWHASH_OPSLIMIT_INTERACTIVE || $wrapped['ops'] > SODIUM_CRYPTO_PWHASH_OPSLIMIT_SENSITIVE
|
|
||||||
|| $wrapped['mem'] < SODIUM_CRYPTO_PWHASH_MEMLIMIT_INTERACTIVE || $wrapped['mem'] > SODIUM_CRYPTO_PWHASH_MEMLIMIT_SENSITIVE) {
|
|
||||||
throw new CryptoException('Unsupported key derivation parameters (damaged or foreign key file)');
|
|
||||||
}
|
|
||||||
$kek = sodium_crypto_pwhash(SODIUM_CRYPTO_SECRETBOX_KEYBYTES, $passphrase, $b64($wrapped['salt']),
|
$kek = sodium_crypto_pwhash(SODIUM_CRYPTO_SECRETBOX_KEYBYTES, $passphrase, $b64($wrapped['salt']),
|
||||||
$wrapped['ops'], $wrapped['mem'], SODIUM_CRYPTO_PWHASH_ALG_ARGON2ID13);
|
$wrapped['ops'], $wrapped['mem'], SODIUM_CRYPTO_PWHASH_ALG_ARGON2ID13);
|
||||||
$master = sodium_crypto_secretbox_open($b64($wrapped['key']), $b64($wrapped['nonce']), $kek);
|
$master = sodium_crypto_secretbox_open($b64($wrapped['key']), $b64($wrapped['nonce']), $kek);
|
||||||
|
|||||||
@@ -122,7 +122,6 @@ final class StreamCipher {
|
|||||||
/** fread() may return short reads on network streams; keep reading until $length or EOF. */
|
/** fread() may return short reads on network streams; keep reading until $length or EOF. */
|
||||||
private static function readFull($in, int $length): string {
|
private static function readFull($in, int $length): string {
|
||||||
$buffer = '';
|
$buffer = '';
|
||||||
$empty = 0;
|
|
||||||
while (strlen($buffer) < $length && !feof($in)) {
|
while (strlen($buffer) < $length && !feof($in)) {
|
||||||
$chunk = fread($in, $length - strlen($buffer));
|
$chunk = fread($in, $length - strlen($buffer));
|
||||||
if ($chunk === false) {
|
if ($chunk === false) {
|
||||||
@@ -133,15 +132,8 @@ final class StreamCipher {
|
|||||||
if (feof($in)) {
|
if (feof($in)) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
// A stalled network stream returns '' without EOF forever: back off, then give up
|
|
||||||
// (about a minute) instead of spinning at full CPU until the lease expires.
|
|
||||||
if (++$empty > 600) {
|
|
||||||
throw new CryptoException('Read stalled');
|
|
||||||
}
|
|
||||||
usleep(100000);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
$empty = 0;
|
|
||||||
$buffer .= $chunk;
|
$buffer .= $chunk;
|
||||||
}
|
}
|
||||||
return $buffer;
|
return $buffer;
|
||||||
|
|||||||
@@ -1,34 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Db;
|
|
||||||
|
|
||||||
use OCA\NgBackup\AppInfo\Application;
|
|
||||||
use OCA\NgBackup\Repository\ICatalogAnchor;
|
|
||||||
use OCP\IAppConfig;
|
|
||||||
|
|
||||||
/** Keeps the newest catalog generation seen per repository in the app config (outside the location). */
|
|
||||||
final class AppConfigCatalogAnchor implements ICatalogAnchor {
|
|
||||||
public function __construct(
|
|
||||||
private IAppConfig $appConfig,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public function get(string $repositoryId): ?array {
|
|
||||||
$raw = $this->appConfig->getValueString(Application::APP_ID, $this->key($repositoryId), '', true);
|
|
||||||
$data = $raw === '' ? null : json_decode($raw, true);
|
|
||||||
return is_array($data) && isset($data['gen'], $data['hash']) ? ['gen' => (int)$data['gen'], 'hash' => (string)$data['hash']] : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function set(string $repositoryId, int $gen, string $hash): void {
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, $this->key($repositoryId), json_encode(['gen' => $gen, 'hash' => $hash]), true);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function key(string $repositoryId): string {
|
|
||||||
return 'catalog_' . preg_replace('/[^a-f0-9]/', '', $repositoryId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+56
-146
@@ -31,8 +31,6 @@ use OCA\NgBackup\Repository\Repository;
|
|||||||
*/
|
*/
|
||||||
final class DbDumper {
|
final class DbDumper {
|
||||||
private const BATCH = 2000;
|
private const BATCH = 2000;
|
||||||
/** Rows per INSERT on restore; keeps the number of bound parameters well below database limits. */
|
|
||||||
private const INSERT_ROWS = 100;
|
|
||||||
/** Tables without a primary key are read in one query; refuse that above this size. */
|
/** Tables without a primary key are read in one query; refuse that above this size. */
|
||||||
private const MAX_ROWS_WITHOUT_PK = 50000;
|
private const MAX_ROWS_WITHOUT_PK = 50000;
|
||||||
|
|
||||||
@@ -43,94 +41,39 @@ final class DbDumper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Dump everything in one go (one transaction, fully consistent).
|
* @return array{tables: array<string, array{blobs: list<string>, rows: int, bytes: int, sha256: string}>, provider: string}
|
||||||
*
|
|
||||||
* @return array{tables: array<string, array{blobs: list<string>, rows: int, bytes: int, sha256: string}>, sequences: array, provider: string, consistent: bool}
|
|
||||||
*/
|
*/
|
||||||
public function dump(Repository $repo, array &$stats): array {
|
public function dump(Repository $repo, array &$stats): array {
|
||||||
$state = $this->startDump();
|
|
||||||
while (!$state['done']) {
|
|
||||||
$state = $this->dumpStep($repo, $state, PHP_FLOAT_MAX, $stats);
|
|
||||||
}
|
|
||||||
return $this->manifest($state);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** State for a resumable dump (see dumpStep()). */
|
|
||||||
public function startDump(): array {
|
|
||||||
$tables = array_values(array_filter($this->db->createSchemaManager()->listTableNames(), fn (string $t) => str_starts_with($t, $this->prefix)));
|
|
||||||
sort($tables);
|
|
||||||
return ['tables' => $tables, 'i' => 0, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null,
|
|
||||||
'result' => [], 'sequences' => [], 'steps' => 0, 'done' => false];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dump tables until $deadline, inside one REPEATABLE READ transaction for this step. A table
|
|
||||||
* that does not fit is resumed in the next step at the last primary key written. If the whole
|
|
||||||
* dump fits in one step it is fully consistent; otherwise each step is consistent on its own
|
|
||||||
* ('consistent' => false in the manifest).
|
|
||||||
*/
|
|
||||||
public function dumpStep(Repository $repo, array $state, float $deadline, array &$stats): array {
|
|
||||||
$state['steps']++;
|
|
||||||
$sm = $this->db->createSchemaManager();
|
$sm = $this->db->createSchemaManager();
|
||||||
// A pack that fills up during the dump is uploaded right away; its index cache row must
|
$tables = array_values(array_filter($sm->listTableNames(), fn (string $t) => str_starts_with($t, $this->prefix)));
|
||||||
// wait until the read-only transaction is over.
|
sort($tables);
|
||||||
$repo->holdIndexCache();
|
|
||||||
|
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
|
||||||
|
$this->db->beginTransaction();
|
||||||
try {
|
try {
|
||||||
$this->db->setTransactionIsolation(TransactionIsolationLevel::REPEATABLE_READ);
|
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
|
||||||
$this->db->beginTransaction();
|
$this->db->executeStatement('SET TRANSACTION READ ONLY');
|
||||||
try {
|
|
||||||
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
|
|
||||||
$this->db->executeStatement('SET TRANSACTION READ ONLY');
|
|
||||||
}
|
|
||||||
while ($state['i'] < count($state['tables'])) {
|
|
||||||
$table = $state['tables'][$state['i']];
|
|
||||||
$schema = $sm->introspectTable($table);
|
|
||||||
$writer = $repo->blobWriter($stats);
|
|
||||||
$hash = $state['hash'] !== null ? unserialize(base64_decode($state['hash']), ['allowed_classes' => [\HashContext::class]]) : hash_init('sha256');
|
|
||||||
if ($state['after'] === null && $state['segments'] === []) {
|
|
||||||
$writer->write($this->header($schema) . "\n");
|
|
||||||
}
|
|
||||||
$complete = true;
|
|
||||||
foreach ($this->rowLines($schema, $state['after']) as [$key, $line]) {
|
|
||||||
$writer->write($line . "\n");
|
|
||||||
hash_update($hash, $line . "\n");
|
|
||||||
$state['rows']++;
|
|
||||||
$state['after'] = $key;
|
|
||||||
if (microtime(true) >= $deadline) {
|
|
||||||
$complete = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$state['segments'][] = $writer->finish();
|
|
||||||
$state['bytes'] += $writer->bytes();
|
|
||||||
if (!$complete) {
|
|
||||||
$state['hash'] = base64_encode(serialize($hash));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
$state['result'][$table] = ['blobs' => array_merge(...$state['segments']), 'rows' => $state['rows'],
|
|
||||||
'bytes' => $state['bytes'], 'sha256' => hash_final($hash)];
|
|
||||||
$state = array_merge($state, ['i' => $state['i'] + 1, 'after' => null, 'segments' => [], 'rows' => 0, 'bytes' => 0, 'hash' => null]);
|
|
||||||
if (microtime(true) >= $deadline && $state['i'] < count($state['tables'])) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($state['i'] >= count($state['tables'])) {
|
|
||||||
$state['sequences'] = $this->sequenceStates($state['tables']);
|
|
||||||
$state['done'] = true;
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
$this->db->rollBack();
|
|
||||||
}
|
}
|
||||||
|
$result = [];
|
||||||
|
foreach ($tables as $table) {
|
||||||
|
$writer = $repo->blobWriter($stats);
|
||||||
|
$hash = hash_init('sha256');
|
||||||
|
$rows = 0;
|
||||||
|
foreach ($this->tableLines($sm->introspectTable($table)) as $kind => $line) {
|
||||||
|
$writer->write($line . "\n");
|
||||||
|
if ($kind === 'row') {
|
||||||
|
hash_update($hash, $line . "\n");
|
||||||
|
$rows++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$result[$table] = ['blobs' => $writer->finish(), 'rows' => $rows, 'bytes' => $writer->bytes(), 'sha256' => hash_final($hash)];
|
||||||
|
}
|
||||||
|
$sequences = $this->sequenceStates($tables);
|
||||||
} finally {
|
} finally {
|
||||||
$repo->releaseIndexCache();
|
$this->db->rollBack();
|
||||||
}
|
}
|
||||||
$repo->flushPacks();
|
$repo->flushPacks();
|
||||||
return $state;
|
return ['tables' => $result, 'sequences' => $sequences, 'provider' => get_class($this->db->getDatabasePlatform())];
|
||||||
}
|
|
||||||
|
|
||||||
public function manifest(array $state): array {
|
|
||||||
return ['tables' => $state['result'], 'sequences' => $state['sequences'],
|
|
||||||
'provider' => get_class($this->db->getDatabasePlatform()), 'consistent' => $state['steps'] === 1];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -139,23 +82,16 @@ final class DbDumper {
|
|||||||
*
|
*
|
||||||
* @param array{tables: array<string, array{blobs: list<string>, rows: int}>} $manifest
|
* @param array{tables: array<string, array{blobs: list<string>, rows: int}>} $manifest
|
||||||
*/
|
*/
|
||||||
public function restore(Repository $repo, array $manifest, ?array &$report = null, ?callable $heartbeat = null): int {
|
public function restore(Repository $repo, array $manifest): int {
|
||||||
$sm = $this->db->createSchemaManager();
|
$sm = $this->db->createSchemaManager();
|
||||||
$existing = array_flip($sm->listTableNames());
|
$existing = array_flip($sm->listTableNames());
|
||||||
$total = 0;
|
$total = 0;
|
||||||
$report = ['skipped' => [], 'restored' => []];
|
|
||||||
$this->db->beginTransaction();
|
$this->db->beginTransaction();
|
||||||
try {
|
try {
|
||||||
foreach ($manifest['tables'] as $table => $info) {
|
foreach ($manifest['tables'] as $table => $info) {
|
||||||
if (!isset($existing[$table])) {
|
if (!isset($existing[$table])) {
|
||||||
throw new \RuntimeException("Table $table from the backup does not exist here (different app versions?)");
|
throw new \RuntimeException("Table $table from the backup does not exist here (different app versions?)");
|
||||||
}
|
}
|
||||||
// Unchanged since the backup: leave it alone (much faster than delete + insert).
|
|
||||||
$live = $this->checksum($table);
|
|
||||||
if ($live['rows'] === $info['rows'] && hash_equals($info['sha256'], $live['sha256'])) {
|
|
||||||
$report['skipped'][] = $table;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$this->db->executeStatement('DELETE FROM ' . $this->db->quoteIdentifier($table));
|
$this->db->executeStatement('DELETE FROM ' . $this->db->quoteIdentifier($table));
|
||||||
$columns = null;
|
$columns = null;
|
||||||
$binary = [];
|
$binary = [];
|
||||||
@@ -176,21 +112,17 @@ final class DbDumper {
|
|||||||
if (is_array($value) && isset($value['$b'])) {
|
if (is_array($value) && isset($value['$b'])) {
|
||||||
$value = base64_decode($value['$b'], true); // text that was not valid UTF-8
|
$value = base64_decode($value['$b'], true); // text that was not valid UTF-8
|
||||||
}
|
}
|
||||||
$row[] = ($value !== null && isset($binary[$col])) ? base64_decode($value, true) : $value;
|
$row[$col] = ($value !== null && isset($binary[$col])) ? base64_decode($value, true) : $value;
|
||||||
}
|
}
|
||||||
$batch[] = $row;
|
$batch[] = $row;
|
||||||
if (count($batch) >= self::INSERT_ROWS) {
|
if (count($batch) >= 500) {
|
||||||
$total += $this->insertBatch($table, $columns, $batch, $binary);
|
$total += $this->insertBatch($table, $batch, $binary);
|
||||||
$batch = [];
|
$batch = [];
|
||||||
if ($heartbeat !== null) {
|
|
||||||
$heartbeat();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ($batch !== []) {
|
if ($batch !== []) {
|
||||||
$total += $this->insertBatch($table, $columns, $batch, $binary);
|
$total += $this->insertBatch($table, $batch, $binary);
|
||||||
}
|
}
|
||||||
$report['restored'][] = $table;
|
|
||||||
}
|
}
|
||||||
$this->restoreSequences($manifest['sequences'] ?? []);
|
$this->restoreSequences($manifest['sequences'] ?? []);
|
||||||
$this->db->commit();
|
$this->db->commit();
|
||||||
@@ -203,22 +135,20 @@ final class DbDumper {
|
|||||||
|
|
||||||
/** Same checksum as dump(): sha256 over the row lines in primary-key order. */
|
/** Same checksum as dump(): sha256 over the row lines in primary-key order. */
|
||||||
public function checksum(string $table): array {
|
public function checksum(string $table): array {
|
||||||
|
$sm = $this->db->createSchemaManager();
|
||||||
$hash = hash_init('sha256');
|
$hash = hash_init('sha256');
|
||||||
$rows = 0;
|
$rows = 0;
|
||||||
foreach ($this->rowLines($this->db->createSchemaManager()->introspectTable($table)) as [, $line]) {
|
foreach ($this->tableLines($sm->introspectTable($table)) as $kind => $line) {
|
||||||
hash_update($hash, $line . "\n");
|
if ($kind === 'row') {
|
||||||
$rows++;
|
hash_update($hash, $line . "\n");
|
||||||
|
$rows++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return ['rows' => $rows, 'sha256' => hash_final($hash)];
|
return ['rows' => $rows, 'sha256' => hash_final($hash)];
|
||||||
}
|
}
|
||||||
|
|
||||||
private function header(\Doctrine\DBAL\Schema\Table $table): string {
|
/** @return \Generator<string, string> 'header' => line, then 'row' => line ... */
|
||||||
[$columns, $binary, $pk] = $this->describe($table);
|
private function tableLines(\Doctrine\DBAL\Schema\Table $table): \Generator {
|
||||||
return json_encode(['table' => $table->getName(), 'columns' => $columns, 'binary' => $binary, 'pk' => $pk], JSON_THROW_ON_ERROR);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return array{0: list<string>, 1: list<string>, 2: list<string>} columns, binary columns, primary key */
|
|
||||||
private function describe(\Doctrine\DBAL\Schema\Table $table): array {
|
|
||||||
$columns = array_values(array_map(fn ($c) => $c->getName(), $table->getColumns()));
|
$columns = array_values(array_map(fn ($c) => $c->getName(), $table->getColumns()));
|
||||||
$binary = [];
|
$binary = [];
|
||||||
foreach ($table->getColumns() as $c) {
|
foreach ($table->getColumns() as $c) {
|
||||||
@@ -226,34 +156,25 @@ final class DbDumper {
|
|||||||
$binary[] = $c->getName();
|
$binary[] = $c->getName();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return [$columns, $binary, $table->getPrimaryKey()?->getColumns() ?? []];
|
$pk = $table->getPrimaryKey()?->getColumns() ?? [];
|
||||||
}
|
yield 'header' => json_encode(['table' => $table->getName(), 'columns' => array_values($columns), 'binary' => $binary, 'pk' => $pk], JSON_THROW_ON_ERROR);
|
||||||
|
|
||||||
/**
|
|
||||||
* Rows in primary-key order, after the key $after (resume), as [key, JSON line].
|
|
||||||
*
|
|
||||||
* @return \Generator<array{0: list<mixed>|null, 1: string}>
|
|
||||||
*/
|
|
||||||
private function rowLines(\Doctrine\DBAL\Schema\Table $table, ?array $after = null): \Generator {
|
|
||||||
[$columns, $binary, $pk] = $this->describe($table);
|
|
||||||
$binaryFlip = array_flip($binary);
|
$binaryFlip = array_flip($binary);
|
||||||
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
||||||
$select = 'SELECT ' . implode(', ', array_map($q, $columns)) . ' FROM ' . $q($table->getName());
|
$select = 'SELECT ' . implode(', ', array_map($q, $columns)) . ' FROM ' . $q($table->getName());
|
||||||
|
|
||||||
if ($pk === []) {
|
if ($pk === []) {
|
||||||
// Nextcloud requires primary keys; small odd tables are read in one query (not resumable).
|
// Nextcloud requires primary keys; small odd tables are read in one query.
|
||||||
$count = (int)$this->db->fetchOne('SELECT COUNT(*) FROM ' . $q($table->getName()));
|
$count = (int)$this->db->fetchOne('SELECT COUNT(*) FROM ' . $q($table->getName()));
|
||||||
if ($count > self::MAX_ROWS_WITHOUT_PK) {
|
if ($count > self::MAX_ROWS_WITHOUT_PK) {
|
||||||
throw new \RuntimeException("Table {$table->getName()} has no primary key and $count rows; cannot dump it with bounded memory");
|
throw new \RuntimeException("Table {$table->getName()} has no primary key and $count rows; cannot dump it with bounded memory");
|
||||||
}
|
}
|
||||||
$batches = [$this->db->fetchAllNumeric($select)];
|
$batches = [$this->db->fetchAllNumeric($select)];
|
||||||
$pkIdx = [];
|
|
||||||
} else {
|
} else {
|
||||||
$batches = $this->keysetBatches($select, $pk, $columns, $after);
|
$batches = $this->keysetBatches($select, $pk, $columns);
|
||||||
$pkIdx = array_map(fn ($c) => array_search($c, $columns, true), $pk);
|
|
||||||
}
|
}
|
||||||
foreach ($batches as $rows) {
|
foreach ($batches as $rows) {
|
||||||
foreach ($rows as $row) {
|
foreach ($rows as $row) {
|
||||||
$key = $pkIdx === [] ? null : array_map(fn ($i) => $row[$i], $pkIdx);
|
|
||||||
foreach ($columns as $i => $col) {
|
foreach ($columns as $i => $col) {
|
||||||
if (is_resource($row[$i])) {
|
if (is_resource($row[$i])) {
|
||||||
$row[$i] = stream_get_contents($row[$i]);
|
$row[$i] = stream_get_contents($row[$i]);
|
||||||
@@ -263,32 +184,28 @@ final class DbDumper {
|
|||||||
} elseif (is_string($row[$i]) && !mb_check_encoding($row[$i], 'UTF-8')) {
|
} elseif (is_string($row[$i]) && !mb_check_encoding($row[$i], 'UTF-8')) {
|
||||||
// Keep invalid UTF-8 byte-exact instead of substituting characters.
|
// Keep invalid UTF-8 byte-exact instead of substituting characters.
|
||||||
$row[$i] = ['$b' => base64_encode($row[$i])];
|
$row[$i] = ['$b' => base64_encode($row[$i])];
|
||||||
} elseif (is_float($row[$i])) {
|
} elseif (is_int($row[$i]) || is_float($row[$i]) || is_bool($row[$i])) {
|
||||||
// var_export uses serialize_precision (shortest exact form), not the lossy
|
|
||||||
// 14-digit 'precision' of a (string) cast.
|
|
||||||
$row[$i] = var_export($row[$i], true);
|
|
||||||
} elseif (is_int($row[$i]) || is_bool($row[$i])) {
|
|
||||||
// Normalise: drivers differ in returning numbers as int or string.
|
// Normalise: drivers differ in returning numbers as int or string.
|
||||||
$row[$i] = (string)(is_bool($row[$i]) ? (int)$row[$i] : $row[$i]);
|
$row[$i] = (string)(is_bool($row[$i]) ? (int)$row[$i] : $row[$i]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
yield [$key, json_encode($row, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR)];
|
yield 'row' => json_encode($row, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return \Generator<list<list<mixed>>> */
|
/** @return \Generator<list<list<mixed>>> */
|
||||||
private function keysetBatches(string $select, array $pk, array $columns, ?array $after): \Generator {
|
private function keysetBatches(string $select, array $pk, array $columns): \Generator {
|
||||||
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
||||||
$order = ' ORDER BY ' . implode(', ', array_map($q, $pk));
|
$order = ' ORDER BY ' . implode(', ', array_map($q, $pk));
|
||||||
$cols = array_map(fn ($c) => $q($c), $pk);
|
$pkIdx = array_map(fn ($c) => array_search($c, $columns, true), $pk);
|
||||||
$last = $after;
|
$last = null;
|
||||||
while (true) {
|
while (true) {
|
||||||
$sql = $select;
|
$sql = $select;
|
||||||
$params = [];
|
$params = [];
|
||||||
if ($last !== null) {
|
if ($last !== null) {
|
||||||
// Row-value comparison (a, b) > (?, ?), supported by PostgreSQL, MySQL/MariaDB and SQLite.
|
// Row-value comparison (a, b) > (?, ?), supported by PostgreSQL, MySQL/MariaDB and SQLite.
|
||||||
$sql .= ' WHERE (' . implode(', ', $cols) . ') > (' . implode(', ', array_fill(0, count($pk), '?')) . ')';
|
$sql .= ' WHERE (' . implode(', ', array_map($q, $pk)) . ') > (' . implode(', ', array_fill(0, count($pk), '?')) . ')';
|
||||||
$params = $last;
|
$params = $last;
|
||||||
}
|
}
|
||||||
$rows = $this->db->fetchAllNumeric($sql . $order . ' LIMIT ' . self::BATCH, $params);
|
$rows = $this->db->fetchAllNumeric($sql . $order . ' LIMIT ' . self::BATCH, $params);
|
||||||
@@ -296,29 +213,22 @@ final class DbDumper {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
yield $rows;
|
yield $rows;
|
||||||
|
$end = end($rows);
|
||||||
|
$last = array_map(fn ($i) => $end[$i], $pkIdx);
|
||||||
if (count($rows) < self::BATCH) {
|
if (count($rows) < self::BATCH) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
$end = end($rows);
|
|
||||||
$last = array_map(fn ($c) => $end[array_search($c, $columns, true)], $pk);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One INSERT with several rows (portable multi-row VALUES). */
|
private function insertBatch(string $table, array $rows, array $binary): int {
|
||||||
private function insertBatch(string $table, array $columns, array $rows, array $binary): int {
|
|
||||||
$q = fn (string $c) => $this->db->quoteIdentifier($c);
|
|
||||||
$placeholders = '(' . implode(', ', array_fill(0, count($columns), '?')) . ')';
|
|
||||||
$sql = 'INSERT INTO ' . $q($table) . ' (' . implode(', ', array_map($q, $columns)) . ') VALUES '
|
|
||||||
. implode(', ', array_fill(0, count($rows), $placeholders));
|
|
||||||
$params = [];
|
|
||||||
$types = [];
|
|
||||||
foreach ($rows as $row) {
|
foreach ($rows as $row) {
|
||||||
foreach ($columns as $i => $col) {
|
$types = [];
|
||||||
$params[] = $row[$i];
|
foreach ($row as $col => $_) {
|
||||||
$types[] = isset($binary[$col]) ? \Doctrine\DBAL\ParameterType::LARGE_OBJECT : ($row[$i] === null ? \Doctrine\DBAL\ParameterType::NULL : \Doctrine\DBAL\ParameterType::STRING);
|
$types[$col] = isset($binary[$col]) ? \Doctrine\DBAL\ParameterType::LARGE_OBJECT : \Doctrine\DBAL\ParameterType::STRING;
|
||||||
}
|
}
|
||||||
|
$this->db->insert($this->db->quoteIdentifier($table), array_combine(array_map(fn ($c) => $this->db->quoteIdentifier($c), array_keys($row)), $row), array_values($types));
|
||||||
}
|
}
|
||||||
$this->db->executeStatement($sql, $params, $types);
|
|
||||||
return count($rows);
|
return count($rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+8
-73
@@ -11,7 +11,7 @@ use OCA\NgBackup\Repository\IIndexCache;
|
|||||||
use OCP\DB\Exception;
|
use OCP\DB\Exception;
|
||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
|
||||||
final class DbIndexCache implements IIndexCache {
|
class DbIndexCache implements IIndexCache {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private IDBConnection $db,
|
private IDBConnection $db,
|
||||||
) {
|
) {
|
||||||
@@ -30,84 +30,19 @@ final class DbIndexCache implements IIndexCache {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function put(string $repositoryId, string $packId, array $entries): void {
|
public function put(string $repositoryId, string $packId, array $entries): void {
|
||||||
try {
|
|
||||||
$this->insert($repositoryId, $packId, $entries);
|
|
||||||
} catch (Exception $e) {
|
|
||||||
if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* In one transaction: on PostgreSQL with its default settings every statement of its own
|
|
||||||
* waits for a disk sync, which made 27,547 rows take more than 6 minutes. Rows already there
|
|
||||||
* are replaced. Should the transaction fail (another process writing the same rows), the
|
|
||||||
* rows are written one by one, keeping rows that are already there.
|
|
||||||
*/
|
|
||||||
public function putMany(string $repositoryId, array $packs): void {
|
|
||||||
if ($packs === []) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$this->transaction(function () use ($repositoryId, $packs): void {
|
|
||||||
$this->remove($repositoryId, array_keys($packs));
|
|
||||||
$this->insertAll($repositoryId, $packs);
|
|
||||||
});
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
if ($this->db->inTransaction()) {
|
|
||||||
throw $e; // a transaction of the caller's, which may be spoilt now
|
|
||||||
}
|
|
||||||
foreach ($packs as $packId => $entries) {
|
|
||||||
$this->put($repositoryId, $packId, $entries);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** All or nothing: when it fails, the cache stays as it was. */
|
|
||||||
public function replaceAll(string $repositoryId, array $packs): void {
|
|
||||||
$this->transaction(function () use ($repositoryId, $packs): void {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->delete('ngb_index')->where($qb->expr()->eq('repository_id', $qb->createNamedParameter($repositoryId)));
|
|
||||||
$qb->executeStatement();
|
|
||||||
$this->insertAll($repositoryId, $packs);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Run $write in a transaction of its own, or as part of the caller's when there is one. */
|
|
||||||
private function transaction(\Closure $write): void {
|
|
||||||
if ($this->db->inTransaction()) {
|
|
||||||
$write();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$this->db->beginTransaction();
|
|
||||||
try {
|
|
||||||
$write();
|
|
||||||
$this->db->commit();
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
if ($this->db->inTransaction()) {
|
|
||||||
$this->db->rollBack();
|
|
||||||
}
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> $packs */
|
|
||||||
private function insertAll(string $repositoryId, array $packs): void {
|
|
||||||
foreach ($packs as $packId => $entries) {
|
|
||||||
$this->insert($repositoryId, $packId, $entries);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
|
||||||
private function insert(string $repositoryId, string $packId, array $entries): void {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
$qb = $this->db->getQueryBuilder();
|
||||||
$qb->insert('ngb_index')->values([
|
$qb->insert('ngb_index')->values([
|
||||||
'repository_id' => $qb->createNamedParameter($repositoryId),
|
'repository_id' => $qb->createNamedParameter($repositoryId),
|
||||||
'pack_id' => $qb->createNamedParameter($packId),
|
'pack_id' => $qb->createNamedParameter($packId),
|
||||||
'entries' => $qb->createNamedParameter(json_encode($entries, JSON_THROW_ON_ERROR)),
|
'entries' => $qb->createNamedParameter(json_encode($entries, JSON_THROW_ON_ERROR)),
|
||||||
]);
|
]);
|
||||||
$qb->executeStatement();
|
try {
|
||||||
|
$qb->executeStatement();
|
||||||
|
} catch (Exception $e) {
|
||||||
|
if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
||||||
|
throw $e;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public function remove(string $repositoryId, array $packIds): void {
|
public function remove(string $repositoryId, array $packIds): void {
|
||||||
|
|||||||
+1
-1
@@ -33,7 +33,7 @@ use OCP\AppFramework\Db\Entity;
|
|||||||
* @method int|null getFinishedAt()
|
* @method int|null getFinishedAt()
|
||||||
* @method void setFinishedAt(?int $v)
|
* @method void setFinishedAt(?int $v)
|
||||||
*/
|
*/
|
||||||
final class Run extends Entity {
|
class Run extends Entity {
|
||||||
public const RUNNING = 'running';
|
public const RUNNING = 'running';
|
||||||
public const DONE = 'done';
|
public const DONE = 'done';
|
||||||
public const FAILED = 'failed';
|
public const FAILED = 'failed';
|
||||||
|
|||||||
+1
-42
@@ -12,41 +12,11 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
|
|||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
|
||||||
/** @template-extends QBMapper<Run> */
|
/** @template-extends QBMapper<Run> */
|
||||||
final class RunMapper extends QBMapper {
|
class RunMapper extends QBMapper {
|
||||||
public function __construct(IDBConnection $db) {
|
public function __construct(IDBConnection $db) {
|
||||||
parent::__construct($db, 'ngb_runs', Run::class);
|
parent::__construct($db, 'ngb_runs', Run::class);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Write $run only if it is still running and unchanged since it was loaded (updated_at
|
|
||||||
* $loadedAt), in one statement: two processes that loaded the same run cannot both write it.
|
|
||||||
* The new updated_at is always later than $loadedAt, so every write is seen as a change.
|
|
||||||
*
|
|
||||||
* @return Run|null the written run, or null when another process wrote it first
|
|
||||||
*/
|
|
||||||
public function updateIfUnchanged(Run $run, int $loadedAt): ?Run {
|
|
||||||
$run->setUpdatedAt(max(time(), $loadedAt + 1));
|
|
||||||
$types = $run->getFieldTypes();
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->update($this->getTableName());
|
|
||||||
foreach (array_keys($run->getUpdatedFields()) as $field) {
|
|
||||||
if ($field === 'id') {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$value = $run->{'get' . ucfirst($field)}();
|
|
||||||
$type = ($types[$field] ?? 'string') === 'integer' ? IQueryBuilder::PARAM_INT : IQueryBuilder::PARAM_STR;
|
|
||||||
$qb->set($run->propertyToColumn($field), $qb->createNamedParameter($value, $value === null ? IQueryBuilder::PARAM_NULL : $type));
|
|
||||||
}
|
|
||||||
$qb->where($qb->expr()->eq('id', $qb->createNamedParameter($run->getId(), IQueryBuilder::PARAM_INT)))
|
|
||||||
->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Run::RUNNING)))
|
|
||||||
->andWhere($qb->expr()->eq('updated_at', $qb->createNamedParameter($loadedAt, IQueryBuilder::PARAM_INT)));
|
|
||||||
if ($qb->executeStatement() !== 1) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$run->resetUpdatedFields();
|
|
||||||
return $run;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function find(int $id): Run {
|
public function find(int $id): Run {
|
||||||
$qb = $this->db->getQueryBuilder();
|
$qb = $this->db->getQueryBuilder();
|
||||||
$qb->select('*')->from($this->getTableName())->where($qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
|
$qb->select('*')->from($this->getTableName())->where($qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
|
||||||
@@ -73,17 +43,6 @@ final class RunMapper extends QBMapper {
|
|||||||
return $this->findEntities($qb);
|
return $this->findEntities($qb);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<Run> runs of $kind started at or after $since, newest first */
|
|
||||||
public function startedSince(int $targetId, string $kind, int $since): array {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->select('*')->from($this->getTableName())
|
|
||||||
->where($qb->expr()->eq('target_id', $qb->createNamedParameter($targetId, IQueryBuilder::PARAM_INT)))
|
|
||||||
->andWhere($qb->expr()->eq('kind', $qb->createNamedParameter($kind)))
|
|
||||||
->andWhere($qb->expr()->gte('started_at', $qb->createNamedParameter($since, IQueryBuilder::PARAM_INT)))
|
|
||||||
->orderBy('id', 'DESC');
|
|
||||||
return $this->findEntities($qb);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function lastFinished(int $targetId, string $kind): ?Run {
|
public function lastFinished(int $targetId, string $kind): ?Run {
|
||||||
$qb = $this->db->getQueryBuilder();
|
$qb = $this->db->getQueryBuilder();
|
||||||
$qb->select('*')->from($this->getTableName())
|
$qb->select('*')->from($this->getTableName())
|
||||||
|
|||||||
+1
-1
@@ -25,7 +25,7 @@ use OCP\AppFramework\Db\Entity;
|
|||||||
* @method int|null getBytes()
|
* @method int|null getBytes()
|
||||||
* @method void setBytes(?int $v)
|
* @method void setBytes(?int $v)
|
||||||
*/
|
*/
|
||||||
final class Snapshot extends Entity {
|
class Snapshot extends Entity {
|
||||||
protected int $targetId = 0;
|
protected int $targetId = 0;
|
||||||
protected string $snapshotId = '';
|
protected string $snapshotId = '';
|
||||||
protected string $kind = '';
|
protected string $kind = '';
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
|
|||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
|
||||||
/** @template-extends QBMapper<Snapshot> */
|
/** @template-extends QBMapper<Snapshot> */
|
||||||
final class SnapshotMapper extends QBMapper {
|
class SnapshotMapper extends QBMapper {
|
||||||
public function __construct(IDBConnection $db) {
|
public function __construct(IDBConnection $db) {
|
||||||
parent::__construct($db, 'ngb_snapshots', Snapshot::class);
|
parent::__construct($db, 'ngb_snapshots', Snapshot::class);
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-9
@@ -22,14 +22,10 @@ use OCP\AppFramework\Db\Entity;
|
|||||||
* @method void setBasePath(string $v)
|
* @method void setBasePath(string $v)
|
||||||
* @method string|null getRepositoryId()
|
* @method string|null getRepositoryId()
|
||||||
* @method void setRepositoryId(?string $v)
|
* @method void setRepositoryId(?string $v)
|
||||||
* @method bool|null getAppendOnly()
|
|
||||||
* @method void setAppendOnly(bool $v)
|
|
||||||
* @method int|null getMaxBytes()
|
|
||||||
* @method void setMaxBytes(?int $v)
|
|
||||||
* @method int getCreatedAt()
|
* @method int getCreatedAt()
|
||||||
* @method void setCreatedAt(int $v)
|
* @method void setCreatedAt(int $v)
|
||||||
*/
|
*/
|
||||||
final class Target extends Entity {
|
class Target extends Entity {
|
||||||
protected string $name = '';
|
protected string $name = '';
|
||||||
protected string $backend = '';
|
protected string $backend = '';
|
||||||
protected string $auth = '';
|
protected string $auth = '';
|
||||||
@@ -37,12 +33,8 @@ final class Target extends Entity {
|
|||||||
protected string $basePath = '';
|
protected string $basePath = '';
|
||||||
protected ?string $repositoryId = null;
|
protected ?string $repositoryId = null;
|
||||||
protected int $createdAt = 0;
|
protected int $createdAt = 0;
|
||||||
protected ?bool $appendOnly = null;
|
|
||||||
protected ?int $maxBytes = null;
|
|
||||||
|
|
||||||
public function __construct() {
|
public function __construct() {
|
||||||
$this->addType('createdAt', 'integer');
|
$this->addType('createdAt', 'integer');
|
||||||
$this->addType('appendOnly', 'boolean');
|
|
||||||
$this->addType('maxBytes', 'integer');
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use OCP\AppFramework\Db\QBMapper;
|
|||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
|
|
||||||
/** @template-extends QBMapper<Target> */
|
/** @template-extends QBMapper<Target> */
|
||||||
final class TargetMapper extends QBMapper {
|
class TargetMapper extends QBMapper {
|
||||||
public function __construct(IDBConnection $db) {
|
public function __construct(IDBConnection $db) {
|
||||||
parent::__construct($db, 'ngb_targets', Target::class);
|
parent::__construct($db, 'ngb_targets', Target::class);
|
||||||
}
|
}
|
||||||
|
|||||||
+16
-108
@@ -49,11 +49,9 @@ final class BackupRun {
|
|||||||
public static function step(Repository $repo, array $state, float $deadline): array {
|
public static function step(Repository $repo, array $state, float $deadline): array {
|
||||||
$state['steps']++;
|
$state['steps']++;
|
||||||
if ($state['phase'] === 'files') {
|
if ($state['phase'] === 'files') {
|
||||||
$stats = $state['stats'];
|
$walker = new TreeWalker($state['roots'], $state['exclude']);
|
||||||
$walker = new TreeWalker($state['roots'], $state['exclude'], function (string $logical) use (&$state, &$stats): void {
|
|
||||||
self::recordUnreadable($state, $stats, $logical);
|
|
||||||
});
|
|
||||||
$parent = new ParentCursor($repo, $state['parent'], $state['parentPos']);
|
$parent = new ParentCursor($repo, $state['parent'], $state['parentPos']);
|
||||||
|
$stats = $state['stats'];
|
||||||
$lines = [];
|
$lines = [];
|
||||||
$finishedWalk = true;
|
$finishedWalk = true;
|
||||||
|
|
||||||
@@ -70,20 +68,15 @@ final class BackupRun {
|
|||||||
$stats['reused']++;
|
$stats['reused']++;
|
||||||
} else {
|
} else {
|
||||||
$cur = $state['cur'];
|
$cur = $state['cur'];
|
||||||
if ($cur !== null && $cur['path'] === $logical && $cur['version'] !== $version) {
|
if ($cur === null || $cur['path'] !== $logical || $cur['version'] !== $version) {
|
||||||
// Changed since the previous step: start over, unless it keeps changing.
|
if ($cur !== null && $cur['path'] === $logical) {
|
||||||
if (self::countRetry($state, $stats, $logical)) {
|
$stats['retried'] = ($stats['retried'] ?? 0) + 1; // changed since the previous step
|
||||||
$cur['version'] = $version;
|
|
||||||
} else {
|
|
||||||
$cur = null;
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if ($cur === null || $cur['path'] !== $logical) {
|
|
||||||
$cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version];
|
$cur = ['path' => $logical, 'offset' => 0, 'blobs' => [], 'version' => $version];
|
||||||
}
|
}
|
||||||
$fh = @fopen($path, 'rb');
|
$fh = fopen($path, 'rb');
|
||||||
if ($fh === false) {
|
if ($fh === false) {
|
||||||
self::recordUnreadable($state, $stats, $logical);
|
$stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1;
|
||||||
$state['cur'] = null;
|
$state['cur'] = null;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -98,17 +91,12 @@ final class BackupRun {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
fclose($fh);
|
fclose($fh);
|
||||||
$now = Repository::fileVersion($path);
|
if (Repository::fileVersion($path) !== $cur['version']) {
|
||||||
if ($now !== $cur['version']) {
|
// Changed while reading: read it again from the start (now or next step).
|
||||||
if (!self::countRetry($state, $stats, $logical)) {
|
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
||||||
// Changed while reading: read it again from the start (now or next step).
|
$state['cur'] = null;
|
||||||
$state['cur'] = null;
|
$finishedWalk = false;
|
||||||
$finishedWalk = false;
|
break;
|
||||||
break;
|
|
||||||
}
|
|
||||||
// Keeps changing (a live database, VM image or log): keep what was read,
|
|
||||||
// like any copy of a file in use, and report it instead of retrying forever.
|
|
||||||
$cur['version'] = $now;
|
|
||||||
}
|
}
|
||||||
if (!$complete) {
|
if (!$complete) {
|
||||||
$state['cur'] = $cur;
|
$state['cur'] = $cur;
|
||||||
@@ -116,7 +104,6 @@ final class BackupRun {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
$blobs = $cur['blobs'];
|
$blobs = $cur['blobs'];
|
||||||
$size = $cur['offset'];
|
|
||||||
$state['cur'] = null;
|
$state['cur'] = null;
|
||||||
$stats['read']++;
|
$stats['read']++;
|
||||||
}
|
}
|
||||||
@@ -146,98 +133,19 @@ final class BackupRun {
|
|||||||
|
|
||||||
if ($state['phase'] === 'finish') {
|
if ($state['phase'] === 'finish') {
|
||||||
$segments = $state['segments'];
|
$segments = $state['segments'];
|
||||||
$repo->putSnapshot($state['snapshot'],
|
$repo->writeSnapshot($state['snapshot'],
|
||||||
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
|
['time' => $state['started'], 'parent' => $state['parent'], 'label' => $state['label'],
|
||||||
'roots' => array_keys($state['roots']), 'stats' => $state['stats'],
|
'roots' => array_keys($state['roots']), 'stats' => $state['stats']] + $state['meta'],
|
||||||
'warnings' => self::warnings($state)] + $state['meta'],
|
|
||||||
(function () use ($repo, $segments) {
|
(function () use ($repo, $segments) {
|
||||||
foreach ($segments as $blobs) {
|
foreach ($segments as $blobs) {
|
||||||
yield from $repo->readLines($blobs);
|
yield from $repo->readLines($blobs);
|
||||||
}
|
}
|
||||||
})());
|
})());
|
||||||
$state['phase'] = 'commit';
|
$state['phase'] = 'done';
|
||||||
}
|
}
|
||||||
return $state;
|
return $state;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Progress of the files phase as a fraction (at most 0.99 until it is done), or null without an
|
|
||||||
* estimate. Bytes of a large file that is still being read count too.
|
|
||||||
*
|
|
||||||
* @return array{fraction: float, approximate: bool}|null
|
|
||||||
*/
|
|
||||||
public static function progress(array $state): ?array {
|
|
||||||
$estimate = $state['estimate'] ?? null;
|
|
||||||
if (($state['phase'] ?? null) !== 'files' || !is_array($estimate) || ($estimate['bytes'] ?? 0) <= 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$done = (int)($state['stats']['bytes'] ?? 0) + (int)($state['cur']['offset'] ?? 0);
|
|
||||||
return ['fraction' => min(0.99, $done / (int)$estimate['bytes']), 'approximate' => (bool)($estimate['approximate'] ?? true)];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* progress() of a run, while it is in its files phase.
|
|
||||||
*
|
|
||||||
* @return array{fraction: float, approximate: bool}|null
|
|
||||||
*/
|
|
||||||
public static function progressOf(\OCA\NgBackup\Db\Run $run): ?array {
|
|
||||||
if ($run->getStatus() !== \OCA\NgBackup\Db\Run::RUNNING || $run->getPhase() !== 'files') {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
} catch (\JsonException) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return is_array($state['files'] ?? null) ? self::progress($state['files']) : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A file or directory that could not be read: it is missing from this snapshot. Remembered (first
|
|
||||||
* 1000, deduplicated across steps) so the run can report it instead of silently succeeding.
|
|
||||||
*/
|
|
||||||
private static function recordUnreadable(array &$state, array &$stats, string $logical): void {
|
|
||||||
if (isset($state['unreadable'][$logical])) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (count($state['unreadable'] ?? []) < 1000) {
|
|
||||||
$state['unreadable'][$logical] = true;
|
|
||||||
}
|
|
||||||
$stats['unreadable'] = ($stats['unreadable'] ?? 0) + 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return array{unreadable: list<string>, changing: list<string>} examples for the report (first 50 each) */
|
|
||||||
public static function warnings(array $state): array {
|
|
||||||
return [
|
|
||||||
'unreadable' => array_slice(array_map('strval', array_keys($state['unreadable'] ?? [])), 0, 50),
|
|
||||||
'changing' => array_slice(array_map('strval', array_values($state['changing'] ?? [])), 0, 50),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Retries of one file that changes while it is read, before its last read is kept anyway. */
|
|
||||||
public const MAX_RETRIES = 3;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Count a retry of $logical. Returns true once it has changed MAX_RETRIES times: the caller then
|
|
||||||
* keeps the data it read, and the file is listed in $state['changing'] (first 50) for the report.
|
|
||||||
*/
|
|
||||||
private static function countRetry(array &$state, array &$stats, string $logical): bool {
|
|
||||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
|
||||||
$n = (($state['retryPath'] ?? null) === $logical ? (int)($state['retryCount'] ?? 0) : 0) + 1;
|
|
||||||
$state['retryPath'] = $logical;
|
|
||||||
$state['retryCount'] = $n;
|
|
||||||
if ($n < self::MAX_RETRIES) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if ($n === self::MAX_RETRIES) {
|
|
||||||
$stats['changing'] = ($stats['changing'] ?? 0) + 1;
|
|
||||||
if (count($state['changing'] ?? []) < 50) {
|
|
||||||
$state['changing'][] = $logical;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function readFull($fh, int $length): string {
|
private static function readFull($fh, int $length): string {
|
||||||
$buffer = '';
|
$buffer = '';
|
||||||
while (strlen($buffer) < $length && !feof($fh)) {
|
while (strlen($buffer) < $length && !feof($fh)) {
|
||||||
|
|||||||
@@ -23,8 +23,6 @@ final class TreeWalker {
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private array $roots,
|
private array $roots,
|
||||||
private array $exclude = [],
|
private array $exclude = [],
|
||||||
/** Called with the logical path of a directory that cannot be listed. */
|
|
||||||
private ?\Closure $onUnreadable = null,
|
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -59,9 +57,6 @@ final class TreeWalker {
|
|||||||
}
|
}
|
||||||
$names = @scandir($abs);
|
$names = @scandir($abs);
|
||||||
if ($names === false) {
|
if ($names === false) {
|
||||||
if ($this->onUnreadable !== null) {
|
|
||||||
($this->onUnreadable)($logical === '' ? '/' : $logical);
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
$names = array_values(array_diff($names, ['.', '..']));
|
$names = array_values(array_diff($names, ['.', '..']));
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ use OCP\DB\Types;
|
|||||||
use OCP\Migration\IOutput;
|
use OCP\Migration\IOutput;
|
||||||
use OCP\Migration\SimpleMigrationStep;
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
final class Version000001Date20261002000000 extends SimpleMigrationStep {
|
class Version000001Date20261002000000 extends SimpleMigrationStep {
|
||||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||||
/** @var ISchemaWrapper $schema */
|
/** @var ISchemaWrapper $schema */
|
||||||
$schema = $schemaClosure();
|
$schema = $schemaClosure();
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ use OCP\Migration\IOutput;
|
|||||||
use OCP\Migration\SimpleMigrationStep;
|
use OCP\Migration\SimpleMigrationStep;
|
||||||
|
|
||||||
/** Local cache of the per-pack index files, so opening a repository does not download them all. */
|
/** Local cache of the per-pack index files, so opening a repository does not download them all. */
|
||||||
final class Version000002Date20261002010000 extends SimpleMigrationStep {
|
class Version000002Date20261002010000 extends SimpleMigrationStep {
|
||||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
||||||
/** @var ISchemaWrapper $schema */
|
/** @var ISchemaWrapper $schema */
|
||||||
$schema = $schemaClosure();
|
$schema = $schemaClosure();
|
||||||
|
|||||||
@@ -1,28 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Migration;
|
|
||||||
|
|
||||||
use Closure;
|
|
||||||
use OCP\DB\ISchemaWrapper;
|
|
||||||
use OCP\DB\Types;
|
|
||||||
use OCP\Migration\IOutput;
|
|
||||||
use OCP\Migration\SimpleMigrationStep;
|
|
||||||
|
|
||||||
/** Append-only locations: NG Backup never deletes anything there (cleanup by the location's own lifecycle rules). */
|
|
||||||
final class Version000003Date20261002020000 extends SimpleMigrationStep {
|
|
||||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
|
||||||
/** @var ISchemaWrapper $schema */
|
|
||||||
$schema = $schemaClosure();
|
|
||||||
$t = $schema->getTable('ngb_targets');
|
|
||||||
if (!$t->hasColumn('append_only')) {
|
|
||||||
$t->addColumn('append_only', Types::BOOLEAN, ['notnull' => false, 'default' => false]);
|
|
||||||
return $schema;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Migration;
|
|
||||||
|
|
||||||
use Closure;
|
|
||||||
use OCP\DB\ISchemaWrapper;
|
|
||||||
use OCP\DB\Types;
|
|
||||||
use OCP\Migration\IOutput;
|
|
||||||
use OCP\Migration\SimpleMigrationStep;
|
|
||||||
|
|
||||||
/** Optional size limit per location (cost control; S3 has no bucket quota). */
|
|
||||||
final class Version000004Date20261002030000 extends SimpleMigrationStep {
|
|
||||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
|
||||||
/** @var ISchemaWrapper $schema */
|
|
||||||
$schema = $schemaClosure();
|
|
||||||
$t = $schema->getTable('ngb_targets');
|
|
||||||
if (!$t->hasColumn('max_bytes')) {
|
|
||||||
$t->addColumn('max_bytes', Types::BIGINT, ['notnull' => false, 'unsigned' => true]);
|
|
||||||
return $schema;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Migration;
|
|
||||||
|
|
||||||
use Closure;
|
|
||||||
use OCP\DB\ISchemaWrapper;
|
|
||||||
use OCP\DB\Types;
|
|
||||||
use OCP\Migration\IOutput;
|
|
||||||
use OCP\Migration\SimpleMigrationStep;
|
|
||||||
|
|
||||||
/** Leases with an expiry: a killed process cannot block a location for longer than its lease. */
|
|
||||||
final class Version000005Date20261002040000 extends SimpleMigrationStep {
|
|
||||||
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
|
|
||||||
/** @var ISchemaWrapper $schema */
|
|
||||||
$schema = $schemaClosure();
|
|
||||||
if ($schema->hasTable('ngb_leases')) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
$t = $schema->createTable('ngb_leases');
|
|
||||||
$t->addColumn('id', Types::STRING, ['notnull' => true, 'length' => 32]);
|
|
||||||
$t->addColumn('name', Types::STRING, ['notnull' => true, 'length' => 128]);
|
|
||||||
$t->addColumn('mode', Types::STRING, ['notnull' => true, 'length' => 1]);
|
|
||||||
$t->addColumn('expires_at', Types::BIGINT, ['notnull' => true, 'unsigned' => true]);
|
|
||||||
$t->setPrimaryKey(['id']);
|
|
||||||
$t->addIndex(['name'], 'ngb_leases_name');
|
|
||||||
return $schema;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Notification;
|
|
||||||
|
|
||||||
use OCA\NgBackup\AppInfo\Application;
|
|
||||||
use OCP\IURLGenerator;
|
|
||||||
use OCP\L10N\IFactory;
|
|
||||||
use OCP\Notification\INotification;
|
|
||||||
use OCP\Notification\INotifier;
|
|
||||||
use OCP\Notification\UnknownNotificationException;
|
|
||||||
|
|
||||||
final class Notifier implements INotifier {
|
|
||||||
public function __construct(
|
|
||||||
private IFactory $l10n,
|
|
||||||
private IURLGenerator $url,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getID(): string {
|
|
||||||
return Application::APP_ID;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getName(): string {
|
|
||||||
return $this->l10n->get(Application::APP_ID)->t('NG Backup');
|
|
||||||
}
|
|
||||||
|
|
||||||
public function prepare(INotification $notification, string $languageCode): INotification {
|
|
||||||
if ($notification->getApp() !== Application::APP_ID) {
|
|
||||||
throw new UnknownNotificationException();
|
|
||||||
}
|
|
||||||
$l = $this->l10n->get(Application::APP_ID, $languageCode);
|
|
||||||
$p = $notification->getSubjectParameters();
|
|
||||||
switch ($notification->getSubject()) {
|
|
||||||
case 'run_failed':
|
|
||||||
$notification->setParsedSubject($p['kind'] === 'restore'
|
|
||||||
? $l->t('Restore from %s failed', [$p['target']])
|
|
||||||
: $l->t('Backup to %s failed', [$p['target']]));
|
|
||||||
$notification->setParsedMessage((string)($p['error'] ?? ''));
|
|
||||||
break;
|
|
||||||
case 'run_warnings':
|
|
||||||
$notification->setParsedSubject($l->t('Backup to %1$s finished with warnings: %2$s unreadable, %3$s changed while being read', [$p['target'], $p['unreadable'], $p['changing']]));
|
|
||||||
$notification->setParsedMessage($l->t('Unreadable files and folders are missing from this backup (check their permissions). Files that kept changing were saved as they were while being read. Examples: %s', [$p['examples'] ?? '']));
|
|
||||||
break;
|
|
||||||
case 'quota_warning':
|
|
||||||
$notification->setParsedSubject($l->t('Backup location %1$s is at %2$s%% of its limit (%3$s of %4$s)', [$p['target'], $p['percent'], $p['used'], $p['limit']]));
|
|
||||||
$notification->setParsedMessage($l->t('Backups stop when the limit is reached. Raise the limit, or let the retention policy remove older backups.'));
|
|
||||||
break;
|
|
||||||
case 'backup_stale':
|
|
||||||
// 'never' was stored by 0.9.4 and earlier
|
|
||||||
$notification->setParsedSubject(in_array($p['since'] ?? '', ['', 'never'], true)
|
|
||||||
? $l->t('No successful backup to %s yet', [$p['target']])
|
|
||||||
: $l->t('No successful backup to %1$s since %2$s', [$p['target'], $p['since']]));
|
|
||||||
$notification->setParsedMessage($l->t('Check the backup settings and the server log.'));
|
|
||||||
break;
|
|
||||||
case 'security_event':
|
|
||||||
$who = (string)($p['actor'] ?? '');
|
|
||||||
$notification->setParsedSubject(match ($p['event'] ?? '') {
|
|
||||||
'kit_downloaded' => $l->t('%s downloaded the backup recovery kit', [$who]),
|
|
||||||
'target_removed' => $l->t('%1$s removed backup location %2$s', [$who, $p['target'] ?? '']),
|
|
||||||
'target_changed' => $l->t('%1$s changed the connection settings of backup location %2$s', [$who, $p['target'] ?? '']),
|
|
||||||
'append_only_off' => $l->t('%1$s turned off append-only for backup location %2$s', [$who, $p['target'] ?? '']),
|
|
||||||
'retention_changed' => $l->t('%1$s changed the backup retention to %2$s', [$who, $p['policy'] ?? '']),
|
|
||||||
'snapshots_forgotten' => $l->t('%1$s moved %2$s snapshots of %3$s to the trash; they are deleted permanently after %4$s unless restored', [$who, $p['count'] ?? '', $p['target'] ?? '', $p['until'] ?? '']),
|
|
||||||
'snapshot_untrashed' => $l->t('%1$s restored snapshot %2$s of %3$s from the trash', [$who, $p['snapshot'] ?? '', $p['target'] ?? '']),
|
|
||||||
'catalog_trusted' => $l->t('%1$s accepted an older state of backup location %2$s (catalog generation %3$s)', [$who, $p['target'] ?? '', $p['generation'] ?? '']),
|
|
||||||
'slot_added' => $l->t('%1$s added backup passphrase slot %2$s (%3$s); download and confirm the new recovery kit', [$who, $p['slot'] ?? '', $p['label'] ?? '']),
|
|
||||||
'slot_replaced' => $l->t('%1$s set a new passphrase for backup key slot %2$s (%3$s); download and confirm the new recovery kit', [$who, $p['slot'] ?? '', $p['label'] ?? '']),
|
|
||||||
'user_export_retention_changed' => $l->t('%1$s changed how many exports per user are kept to %2$s', [$who, $p['keep'] ?? '']),
|
|
||||||
'user_exports_forgotten' => $l->t('%1$s removed %2$s per-user exports from %3$s', [$who, $p['count'] ?? '', $p['target'] ?? '']),
|
|
||||||
'slot_removed' => $l->t('%1$s removed backup passphrase slot %2$s; download and confirm the new recovery kit', [$who, $p['slot'] ?? '']),
|
|
||||||
default => $l->t('NG Backup security event'),
|
|
||||||
});
|
|
||||||
$notification->setParsedMessage($l->t('If this was not you or another administrator, check who has admin access.'));
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new UnknownNotificationException();
|
|
||||||
}
|
|
||||||
$notification->setLink($this->url->linkToRouteAbsolute('settings.AdminSettings.index', ['section' => 'ng_backup']));
|
|
||||||
$notification->setIcon($this->url->getAbsoluteURL($this->url->imagePath(Application::APP_ID, 'app-dark.svg')));
|
|
||||||
return $notification;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+14
-139
@@ -25,11 +25,7 @@ final class BlobIndex {
|
|||||||
private array $pending = [];
|
private array $pending = [];
|
||||||
|
|
||||||
private ?IIndexCache $cache = null;
|
private ?IIndexCache $cache = null;
|
||||||
/** @var list<\Closure():void>|null cache writes held back by holdCache(), or null when writing directly */
|
|
||||||
private ?array $heldCacheWrites = null;
|
|
||||||
private string $repositoryId = '';
|
private string $repositoryId = '';
|
||||||
/** Index files written to the cache per database transaction while loading. */
|
|
||||||
private const CACHE_LOT = 256;
|
|
||||||
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
|
/** @var int index files downloaded by the last load() (for tests and diagnostics) */
|
||||||
public int $downloaded = 0;
|
public int $downloaded = 0;
|
||||||
|
|
||||||
@@ -44,106 +40,30 @@ final class BlobIndex {
|
|||||||
$this->repositoryId = $repositoryId;
|
$this->repositoryId = $repositoryId;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Hold back cache writes until releaseCache(). The database dump reads inside a read-only
|
|
||||||
* transaction on the same connection that the cache writes to: a write there fails on
|
|
||||||
* PostgreSQL and would be rolled back with the dump transaction elsewhere.
|
|
||||||
*/
|
|
||||||
public function holdCache(): void {
|
|
||||||
$this->heldCacheWrites ??= [];
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Write the cache changes held back since holdCache(). */
|
|
||||||
public function releaseCache(): void {
|
|
||||||
$writes = $this->heldCacheWrites ?? [];
|
|
||||||
$this->heldCacheWrites = null;
|
|
||||||
foreach ($writes as $write) {
|
|
||||||
$write();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
|
||||||
private function cachePut(string $packId, array $entries): void {
|
|
||||||
if ($this->heldCacheWrites !== null) {
|
|
||||||
$this->heldCacheWrites[] = function () use ($packId, $entries): void {
|
|
||||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
|
||||||
};
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$this->cache?->put($this->repositoryId, $packId, $entries);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param list<string> $packIds */
|
|
||||||
private function cacheRemove(array $packIds): void {
|
|
||||||
if ($this->heldCacheWrites !== null) {
|
|
||||||
$this->heldCacheWrites[] = function () use ($packIds): void {
|
|
||||||
$this->cache?->remove($this->repositoryId, $packIds);
|
|
||||||
};
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$this->cache?->remove($this->repositoryId, $packIds);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load the index: one listing of index/, entries from the local cache where available,
|
* Load the index: one listing of index/, entries from the local cache where available,
|
||||||
* and only the index files that are not cached yet are downloaded. Cached packs that no
|
* and only the index files that are not cached yet are downloaded. Cached packs that no
|
||||||
* longer exist on the backend are dropped from the cache.
|
* longer exist on the backend are dropped from the cache.
|
||||||
*/
|
*/
|
||||||
public function load(bool $verify = false, ?callable $progress = null): void {
|
public function load(): void {
|
||||||
$this->blobs = [];
|
$cached = $this->cache?->all($this->repositoryId) ?? [];
|
||||||
$cached = $verify ? [] : ($this->cache?->all($this->repositoryId) ?? []);
|
|
||||||
$present = [];
|
$present = [];
|
||||||
$this->downloaded = 0;
|
$this->downloaded = 0;
|
||||||
$order = []; // pack ids in listing order: where a blob is in two packs, the last listed wins
|
foreach ($this->backend->list('index') as $path) {
|
||||||
$fetch = [];
|
|
||||||
foreach ($this->backend->listFiles('index') as $path) {
|
|
||||||
$packId = basename($path);
|
$packId = basename($path);
|
||||||
$present[$packId] = true;
|
$present[$packId] = true;
|
||||||
$order[] = $packId;
|
if (isset($cached[$packId])) {
|
||||||
if (!isset($cached[$packId])) {
|
$entries = $cached[$packId];
|
||||||
$fetch[] = $path;
|
} else {
|
||||||
|
$fh = $this->backend->get($path);
|
||||||
|
$entries = json_decode($this->cipher->decryptString((string)stream_get_contents($fh), 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
fclose($fh);
|
||||||
|
$this->downloaded++;
|
||||||
|
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||||
}
|
}
|
||||||
}
|
foreach ($entries as $e) {
|
||||||
// The rest is downloaded several at a time where the location allows it: on a large
|
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||||
// repository these are tens of thousands of small files. getMany() keeps their order, so
|
|
||||||
// the cached packs listed before each one are added first.
|
|
||||||
$next = 0;
|
|
||||||
$total = count($fetch);
|
|
||||||
$toCache = []; // written to the cache a few hundred at a time, each lot in one transaction
|
|
||||||
foreach ($this->backend->getMany($fetch) as $path => $data) {
|
|
||||||
$packId = basename($path);
|
|
||||||
$entries = json_decode($this->cipher->decryptString($data, 'index:' . $packId), true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
$this->downloaded++;
|
|
||||||
if ($this->cache !== null) {
|
|
||||||
$toCache[$packId] = $entries;
|
|
||||||
// a check writes the whole cache afresh at the end
|
|
||||||
if (!$verify && count($toCache) >= self::CACHE_LOT) {
|
|
||||||
$this->cache->putMany($this->repositoryId, $toCache);
|
|
||||||
$toCache = [];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for (; $next < count($order) && $order[$next] !== $packId; $next++) {
|
|
||||||
$this->add($order[$next], $cached[$order[$next]] ?? []);
|
|
||||||
}
|
|
||||||
$this->add($packId, $entries);
|
|
||||||
$next++;
|
|
||||||
if ($progress !== null) {
|
|
||||||
$progress($this->downloaded, $total);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (; $next < count($order); $next++) {
|
|
||||||
$this->add($order[$next], $cached[$order[$next]] ?? []);
|
|
||||||
}
|
|
||||||
if ($verify && $this->cache !== null) {
|
|
||||||
// Refresh the cache from what was just verified, as the index files list it: a blob in
|
|
||||||
// two packs is in both rows, so that removing one of the packs leaves the other right.
|
|
||||||
// In one transaction: one row at a time took more than 6 minutes on a 1 TB repository
|
|
||||||
// (27,547 packs), long enough for the location to drop the connection.
|
|
||||||
$this->cache->replaceAll($this->repositoryId, $toCache);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if ($toCache !== []) {
|
|
||||||
$this->cache?->putMany($this->repositoryId, $toCache);
|
|
||||||
}
|
}
|
||||||
$stale = array_keys(array_diff_key($cached, $present));
|
$stale = array_keys(array_diff_key($cached, $present));
|
||||||
if ($stale !== []) {
|
if ($stale !== []) {
|
||||||
@@ -151,13 +71,6 @@ final class BlobIndex {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
|
||||||
private function add(string $packId, array $entries): void {
|
|
||||||
foreach ($entries as $e) {
|
|
||||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function has(string $id): bool {
|
public function has(string $id): bool {
|
||||||
return isset($this->blobs[$id]) || isset($this->pending[$id]);
|
return isset($this->blobs[$id]) || isset($this->pending[$id]);
|
||||||
}
|
}
|
||||||
@@ -184,51 +97,13 @@ final class BlobIndex {
|
|||||||
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
|
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
|
||||||
rewind($stream);
|
rewind($stream);
|
||||||
$this->backend->put('index/' . $packId, $stream);
|
$this->backend->put('index/' . $packId, $stream);
|
||||||
$this->cachePut($packId, $entries);
|
$this->cache?->put($this->repositoryId, $packId, $entries);
|
||||||
foreach ($entries as $e) {
|
foreach ($entries as $e) {
|
||||||
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
|
||||||
unset($this->pending[$e['id']]);
|
unset($this->pending[$e['id']]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return array<string, list<array{id:string, offset:int, length:int}>> packId => blobs currently located in that pack */
|
|
||||||
public function packs(): array {
|
|
||||||
$packs = [];
|
|
||||||
foreach ($this->blobs as $id => [$pack, $offset, $length]) {
|
|
||||||
$packs[$pack][] = ['id' => $id, 'offset' => $offset, 'length' => $length];
|
|
||||||
}
|
|
||||||
return $packs;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Forget a pack: delete its index file and cache row. Blobs that now live elsewhere are kept. */
|
|
||||||
public function removePack(string $packId): void {
|
|
||||||
$this->backend->delete('index/' . $packId);
|
|
||||||
$this->cacheRemove([$packId]);
|
|
||||||
foreach ($this->blobs as $id => $loc) {
|
|
||||||
if ($loc[0] === $packId) {
|
|
||||||
unset($this->blobs[$id]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Number of packs that hold blobs of the index. */
|
|
||||||
public function packCount(): int {
|
|
||||||
$packs = [];
|
|
||||||
foreach ($this->blobs as [$pack]) {
|
|
||||||
$packs[$pack] = true;
|
|
||||||
}
|
|
||||||
return count($packs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Bytes of all blobs in the index (encrypted size, i.e. what is stored in packs). */
|
|
||||||
public function storedBytes(): int {
|
|
||||||
$total = 0;
|
|
||||||
foreach ($this->blobs as [, , $length]) {
|
|
||||||
$total += $length;
|
|
||||||
}
|
|
||||||
return $total;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function count(): int {
|
public function count(): int {
|
||||||
return count($this->blobs);
|
return count($this->blobs);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,141 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Repository;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Backend\IBackend;
|
|
||||||
use OCA\NgBackup\Crypto\StreamCipher;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Chained catalog of a repository's state, against rollback and hidden snapshots.
|
|
||||||
*
|
|
||||||
* Every change to the set of snapshots or trash entries writes catalog/<generation>: the
|
|
||||||
* encrypted list of snapshot and trash ids plus the hash of the previous generation. The server
|
|
||||||
* keeps the newest generation it has seen (ICatalogAnchor). On open, the newest catalog on the
|
|
||||||
* location must be at least the anchored generation and lead back to it through the chain;
|
|
||||||
* snapshots listed in the catalog must exist. Otherwise the location was rolled back or
|
|
||||||
* tampered with, and the repository is refused.
|
|
||||||
*/
|
|
||||||
final class Catalog {
|
|
||||||
private const KEEP = 200; // generations kept on the location (enough to verify long gaps)
|
|
||||||
|
|
||||||
public int $gen = 0;
|
|
||||||
/** @var array<string, true> */
|
|
||||||
public array $snapshots = [];
|
|
||||||
/** @var array<string, true> */
|
|
||||||
public array $trash = [];
|
|
||||||
/** @var array<string, true> manifest path => true, for a per-user export (users/<uid>/<exportId>) */
|
|
||||||
public array $userExports = [];
|
|
||||||
public string $hash = '';
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private IBackend $backend,
|
|
||||||
private StreamCipher $cipher,
|
|
||||||
private string $repositoryId,
|
|
||||||
private ?ICatalogAnchor $anchor,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Load and verify the newest catalog; create the first one for repositories that have none. */
|
|
||||||
public function load(callable $listSnapshots, callable $listTrash, callable $listUserExports): void {
|
|
||||||
$gens = $this->generations();
|
|
||||||
if ($gens === []) {
|
|
||||||
// First use (new or pre-catalog repository): adopt the current state as generation 1.
|
|
||||||
$anchored = $this->anchor?->get($this->repositoryId);
|
|
||||||
if ($anchored !== null) {
|
|
||||||
throw new RepositoryException('The catalog of this repository has disappeared from the location (possible tampering)');
|
|
||||||
}
|
|
||||||
$this->snapshots = array_fill_keys($listSnapshots(), true);
|
|
||||||
$this->trash = array_fill_keys($listTrash(), true);
|
|
||||||
$this->userExports = array_fill_keys($listUserExports(), true);
|
|
||||||
$this->write();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$head = max($gens);
|
|
||||||
[$data, $hash] = $this->read($head);
|
|
||||||
$anchored = $this->anchor?->get($this->repositoryId);
|
|
||||||
if ($anchored !== null) {
|
|
||||||
if ($head < $anchored['gen']) {
|
|
||||||
throw new RepositoryException(sprintf('The location shows catalog generation %d, but this server has seen %d: the backups were rolled back', $head, $anchored['gen']));
|
|
||||||
}
|
|
||||||
// Walk back to the anchored generation, checking every link.
|
|
||||||
[$cur, $curHash] = [$data, $hash];
|
|
||||||
for ($g = $head; $g > $anchored['gen']; $g--) {
|
|
||||||
[$prev, $prevHash] = $this->read($g - 1);
|
|
||||||
if (!hash_equals($prevHash, (string)$cur['prev'])) {
|
|
||||||
throw new RepositoryException("Catalog chain broken at generation $g (possible tampering)");
|
|
||||||
}
|
|
||||||
[$cur, $curHash] = [$prev, $prevHash];
|
|
||||||
}
|
|
||||||
if (!hash_equals($anchored['hash'], $curHash)) {
|
|
||||||
throw new RepositoryException('The catalog does not match what this server saw before (possible tampering)');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$this->gen = $head;
|
|
||||||
$this->hash = $hash;
|
|
||||||
$this->snapshots = array_fill_keys($data['snapshots'], true);
|
|
||||||
$this->trash = array_fill_keys($data['trash'], true);
|
|
||||||
// Catalogs written before user exports existed have no such key: default to empty rather
|
|
||||||
// than adopting the location's current users/ listing (that would skip verification).
|
|
||||||
$this->userExports = array_fill_keys($data['userExports'] ?? [], true);
|
|
||||||
$this->anchor?->set($this->repositoryId, $this->gen, $this->hash);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Write the next generation after the in-memory state was changed. */
|
|
||||||
public function write(): void {
|
|
||||||
$next = $this->gen + 1;
|
|
||||||
// Two writers starting from the same generation would otherwise overwrite each other
|
|
||||||
// silently and break the chain for whichever server anchored the lost write.
|
|
||||||
if ($this->backend->exists($this->path($next))) {
|
|
||||||
throw new RepositoryException("Catalog generation $next was written by another process at the same time; try again");
|
|
||||||
}
|
|
||||||
$plain = json_encode(['repository' => $this->repositoryId, 'gen' => $next, 'time' => gmdate('c'), 'prev' => $this->hash,
|
|
||||||
'snapshots' => array_keys($this->snapshots), 'trash' => array_keys($this->trash),
|
|
||||||
'userExports' => array_keys($this->userExports)], JSON_THROW_ON_ERROR);
|
|
||||||
$encrypted = $this->cipher->encryptString($plain, $this->ad($next));
|
|
||||||
$stream = fopen('php://memory', 'w+b');
|
|
||||||
fwrite($stream, $encrypted);
|
|
||||||
rewind($stream);
|
|
||||||
$this->backend->put($this->path($next), $stream);
|
|
||||||
$this->gen = $next;
|
|
||||||
$this->hash = hash('sha256', $encrypted);
|
|
||||||
$this->anchor?->set($this->repositoryId, $this->gen, $this->hash);
|
|
||||||
$old = $next - self::KEEP;
|
|
||||||
if ($old > 0) {
|
|
||||||
$this->backend->delete($this->path($old));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return array{0: array, 1: string} decoded catalog and the hash of its encrypted bytes */
|
|
||||||
private function read(int $gen): array {
|
|
||||||
try {
|
|
||||||
$fh = $this->backend->get($this->path($gen));
|
|
||||||
} catch (\Throwable) {
|
|
||||||
throw new RepositoryException("Catalog generation $gen is missing (possible tampering)");
|
|
||||||
}
|
|
||||||
$encrypted = (string)stream_get_contents($fh);
|
|
||||||
fclose($fh);
|
|
||||||
$data = json_decode($this->cipher->decryptString($encrypted, $this->ad($gen)), true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
if (($data['repository'] ?? '') !== $this->repositoryId || ($data['gen'] ?? 0) !== $gen) {
|
|
||||||
throw new RepositoryException("Catalog generation $gen belongs to another repository or position");
|
|
||||||
}
|
|
||||||
return [$data, hash('sha256', $encrypted)];
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return list<int> */
|
|
||||||
private function generations(): array {
|
|
||||||
return array_map(fn ($p) => (int)basename($p), $this->backend->listFiles('catalog'));
|
|
||||||
}
|
|
||||||
|
|
||||||
private function path(int $gen): string {
|
|
||||||
return sprintf('catalog/%010d', $gen);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function ad(int $gen): string {
|
|
||||||
return 'catalog:' . $this->repositoryId . ':' . $gen;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Repository;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Where the server remembers the newest catalog generation it has seen for a repository, outside
|
|
||||||
* the (untrusted) location. A location that presents an older generation, or a chain that does
|
|
||||||
* not lead back to this anchor, is rolled back or tampered with.
|
|
||||||
*/
|
|
||||||
interface ICatalogAnchor {
|
|
||||||
/** @return array{gen:int, hash:string}|null */
|
|
||||||
public function get(string $repositoryId): ?array;
|
|
||||||
|
|
||||||
public function set(string $repositoryId, int $gen, string $hash): void;
|
|
||||||
}
|
|
||||||
@@ -15,20 +15,6 @@ interface IIndexCache {
|
|||||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||||
public function put(string $repositoryId, string $packId, array $entries): void;
|
public function put(string $repositoryId, string $packId, array $entries): void;
|
||||||
|
|
||||||
/**
|
|
||||||
* put() for many packs at once (one database transaction).
|
|
||||||
*
|
|
||||||
* @param array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> $packs packId => entries
|
|
||||||
*/
|
|
||||||
public function putMany(string $repositoryId, array $packs): void;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Replace everything cached for the repository with $packs (one database transaction).
|
|
||||||
*
|
|
||||||
* @param array<string, list<array{id:string, offset:int, length:int, raw:int, flags:int}>> $packs packId => entries
|
|
||||||
*/
|
|
||||||
public function replaceAll(string $repositoryId, array $packs): void;
|
|
||||||
|
|
||||||
/** @param list<string> $packIds */
|
/** @param list<string> $packIds */
|
||||||
public function remove(string $repositoryId, array $packIds): void;
|
public function remove(string $repositoryId, array $packIds): void;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,97 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Repository;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Backend\IBackend;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads blobs from packs. Restores and deep verifies read the blobs of a snapshot in the order
|
|
||||||
* they were written, so the stream of the pack read last is kept open and read on from where it
|
|
||||||
* is: one request per pack instead of one per blob (a round trip each, and on most remote
|
|
||||||
* storages a new connection or file handle too). Anything else opens the pack at the offset,
|
|
||||||
* as a single read always did.
|
|
||||||
*
|
|
||||||
* The open stream must not overlap other requests on the location (SFTP cannot read and write
|
|
||||||
* on one connection at once): Repository reaches the location through a PackReaderGuard, which
|
|
||||||
* closes it first.
|
|
||||||
*/
|
|
||||||
final class PackReader {
|
|
||||||
/** A gap up to this size in the open stream is read past instead of opening the pack again. */
|
|
||||||
private const SKIP_MAX = 8 * 1048576;
|
|
||||||
private const CHUNK = 1048576;
|
|
||||||
|
|
||||||
/** @var resource|null */
|
|
||||||
private $stream = null;
|
|
||||||
private string $pack = '';
|
|
||||||
private int $pos = 0;
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private IBackend $backend,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public function read(string $pack, int $offset, int $length): string {
|
|
||||||
if ($this->stream !== null && $this->pack === $pack && $offset >= $this->pos && $offset - $this->pos <= self::SKIP_MAX) {
|
|
||||||
try {
|
|
||||||
$this->skipTo($offset);
|
|
||||||
return $this->readExactly($length);
|
|
||||||
} catch (\Exception) {
|
|
||||||
// The open stream broke (e.g. a timeout); open the pack again below.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$this->close();
|
|
||||||
$fh = $this->backend->get('packs/' . substr($pack, 0, 2) . '/' . $pack);
|
|
||||||
$this->stream = $fh;
|
|
||||||
$this->pack = $pack;
|
|
||||||
$this->pos = 0;
|
|
||||||
try {
|
|
||||||
if ($offset > 0 && (stream_get_meta_data($fh)['seekable'] ?? false) && fseek($fh, $offset) === 0) {
|
|
||||||
$this->pos = $offset;
|
|
||||||
}
|
|
||||||
$this->skipTo($offset); // a stream that cannot seek is read up to the offset
|
|
||||||
return $this->readExactly($length);
|
|
||||||
} catch (\Exception $e) {
|
|
||||||
$this->close();
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function close(): void {
|
|
||||||
$fh = $this->stream;
|
|
||||||
$this->stream = null;
|
|
||||||
if ($fh !== null) {
|
|
||||||
@fclose($fh);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function __destruct() {
|
|
||||||
$this->close();
|
|
||||||
}
|
|
||||||
|
|
||||||
private function skipTo(int $offset): void {
|
|
||||||
while ($this->pos < $offset) {
|
|
||||||
$this->readExactly(min($offset - $this->pos, self::CHUNK));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Read $length bytes from the open stream; a short read closes it. */
|
|
||||||
private function readExactly(int $length): string {
|
|
||||||
$fh = $this->stream ?? throw new RepositoryException('No open pack stream');
|
|
||||||
$data = '';
|
|
||||||
while (strlen($data) < $length) {
|
|
||||||
$chunk = fread($fh, min($length - strlen($data), self::CHUNK));
|
|
||||||
if ($chunk === false || $chunk === '') {
|
|
||||||
$this->close();
|
|
||||||
throw new RepositoryException('Short read in pack ' . $this->pack);
|
|
||||||
}
|
|
||||||
$data .= $chunk;
|
|
||||||
}
|
|
||||||
$this->pos += $length;
|
|
||||||
return $data;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Repository;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Backend\IBackend;
|
|
||||||
|
|
||||||
/** The location as Repository sees it: closes the PackReader's open stream before every other request. */
|
|
||||||
final class PackReaderGuard implements IBackend {
|
|
||||||
public function __construct(
|
|
||||||
private IBackend $inner,
|
|
||||||
private PackReader $reader,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @param resource $stream */
|
|
||||||
public function put(string $path, $stream): void {
|
|
||||||
$this->reader->close();
|
|
||||||
$this->inner->put($path, $stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function get(string $path) {
|
|
||||||
$this->reader->close();
|
|
||||||
return $this->inner->get($path);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getMany(array $paths): \Generator {
|
|
||||||
$this->reader->close();
|
|
||||||
return $this->inner->getMany($paths);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getRange(string $path, int $offset, int $length): string {
|
|
||||||
$this->reader->close();
|
|
||||||
return $this->inner->getRange($path, $offset, $length);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function exists(string $path): bool {
|
|
||||||
$this->reader->close();
|
|
||||||
return $this->inner->exists($path);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function move(string $from, string $to): void {
|
|
||||||
$this->reader->close();
|
|
||||||
$this->inner->move($from, $to);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function list(string $prefix): array {
|
|
||||||
$this->reader->close();
|
|
||||||
return $this->inner->list($prefix);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function listFiles(string $dir): array {
|
|
||||||
$this->reader->close();
|
|
||||||
return $this->inner->listFiles($dir);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function delete(string $path): void {
|
|
||||||
$this->reader->close();
|
|
||||||
$this->inner->delete($path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -32,7 +32,6 @@ final class PackWriter {
|
|||||||
private StreamCipher $cipher,
|
private StreamCipher $cipher,
|
||||||
private BlobIndex $index,
|
private BlobIndex $index,
|
||||||
private int $targetSize = 32 * 1048576,
|
private int $targetSize = 32 * 1048576,
|
||||||
private ?int $maxBytes = null,
|
|
||||||
) {
|
) {
|
||||||
$this->reset();
|
$this->reset();
|
||||||
}
|
}
|
||||||
@@ -50,27 +49,11 @@ final class PackWriter {
|
|||||||
return strlen($encrypted);
|
return strlen($encrypted);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Add an already encrypted blob as-is (repacking: no decryption needed, the blob is bound to its id, not to a pack). */
|
|
||||||
public function addRaw(string $id, string $encrypted, int $rawLength, int $flags): void {
|
|
||||||
fwrite($this->buffer, $encrypted);
|
|
||||||
$this->entries[] = ['id' => $id, 'offset' => $this->size, 'length' => strlen($encrypted), 'raw' => $rawLength, 'flags' => $flags];
|
|
||||||
$this->size += strlen($encrypted);
|
|
||||||
if ($this->size >= $this->targetSize) {
|
|
||||||
$this->flush();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Upload the current pack (if any) and record it in the index. */
|
/** Upload the current pack (if any) and record it in the index. */
|
||||||
public function flush(): void {
|
public function flush(): void {
|
||||||
if ($this->entries === []) {
|
if ($this->entries === []) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if ($this->maxBytes !== null && $this->index->storedBytes() + $this->size > $this->maxBytes) {
|
|
||||||
$this->index->dropPending(array_column($this->entries, 'id'));
|
|
||||||
$this->reset();
|
|
||||||
throw new QuotaExceededException(sprintf('Storage limit of this location reached (%.1f GiB): backup stopped',
|
|
||||||
$this->maxBytes / 1073741824));
|
|
||||||
}
|
|
||||||
$packId = bin2hex(random_bytes(16));
|
$packId = bin2hex(random_bytes(16));
|
||||||
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
|
$header = $this->cipher->encryptString(json_encode($this->entries, JSON_THROW_ON_ERROR), 'pack-header:' . $packId);
|
||||||
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Repository;
|
|
||||||
|
|
||||||
final class QuotaExceededException extends RepositoryException {
|
|
||||||
}
|
|
||||||
+22
-591
@@ -28,17 +28,10 @@ final class Repository {
|
|||||||
public const FORMAT = 1;
|
public const FORMAT = 1;
|
||||||
public const BLOB_SIZE = 4 * 1048576;
|
public const BLOB_SIZE = 4 * 1048576;
|
||||||
private const FLAG_DEFLATE = 1;
|
private const FLAG_DEFLATE = 1;
|
||||||
/** Reads per blob in a deep verify before it is reported as unreadable (not corrupted). */
|
|
||||||
private const VERIFY_READ_ATTEMPTS = 3;
|
|
||||||
|
|
||||||
private StreamCipher $cipher;
|
private StreamCipher $cipher;
|
||||||
private BlobIndex $index;
|
private BlobIndex $index;
|
||||||
private ?PackWriter $runPacks = null;
|
private ?PackWriter $runPacks = null;
|
||||||
private ?int $maxBytes = null;
|
|
||||||
private Catalog $catalog;
|
|
||||||
private PackReader $packReader;
|
|
||||||
/** verify() lists the pack folders from this many packs on (see presentPacks()); tests lower it. */
|
|
||||||
public int $listPacksAbove = 256;
|
|
||||||
|
|
||||||
private function __construct(
|
private function __construct(
|
||||||
private IBackend $backend,
|
private IBackend $backend,
|
||||||
@@ -46,9 +39,7 @@ final class Repository {
|
|||||||
private string $repositoryId,
|
private string $repositoryId,
|
||||||
) {
|
) {
|
||||||
$this->cipher = new StreamCipher($keys);
|
$this->cipher = new StreamCipher($keys);
|
||||||
$this->packReader = new PackReader($backend);
|
$this->index = new BlobIndex($backend, $this->cipher);
|
||||||
$this->backend = new PackReaderGuard($backend, $this->packReader);
|
|
||||||
$this->index = new BlobIndex($this->backend, $this->cipher);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
public static function init(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||||
@@ -61,7 +52,7 @@ final class Repository {
|
|||||||
* copy (KeyRing::wrap) stored in the repository config, so the repository can be opened
|
* copy (KeyRing::wrap) stored in the repository config, so the repository can be opened
|
||||||
* with the passphrase from the recovery kit even when the server is gone.
|
* with the passphrase from the recovery kit even when the server is gone.
|
||||||
*/
|
*/
|
||||||
public static function initWithKey(IBackend $backend, KeyRing $keys, array $wrappedKey, ?ICatalogAnchor $anchor = null): self {
|
public static function initWithKey(IBackend $backend, KeyRing $keys, array $wrappedKey): self {
|
||||||
if ($backend->exists('config')) {
|
if ($backend->exists('config')) {
|
||||||
throw new RepositoryException('A repository already exists at this location');
|
throw new RepositoryException('A repository already exists at this location');
|
||||||
}
|
}
|
||||||
@@ -69,13 +60,11 @@ final class Repository {
|
|||||||
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
|
$config = ['format' => self::FORMAT, 'id' => $id, 'created' => gmdate('c'), 'key' => $wrappedKey,
|
||||||
'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))];
|
'check' => base64_encode((new StreamCipher($keys))->encryptString($id, 'config-check'))];
|
||||||
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
self::putString($backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
||||||
$repo = new self($backend, $keys, $id);
|
return new self($backend, $keys, $id);
|
||||||
$repo->loadCatalog($anchor);
|
|
||||||
return $repo;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
|
/** Open with the master key the server holds (scheduled backups, no passphrase needed). */
|
||||||
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null, ?ICatalogAnchor $anchor = null): self {
|
public static function openWithKey(IBackend $backend, KeyRing $keys, ?IIndexCache $cache = null): self {
|
||||||
$config = self::readConfig($backend);
|
$config = self::readConfig($backend);
|
||||||
try {
|
try {
|
||||||
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
|
$ok = isset($config['check']) && (new StreamCipher($keys))->decryptString((string)base64_decode($config['check'], true), 'config-check') === $config['id'];
|
||||||
@@ -90,32 +79,9 @@ final class Repository {
|
|||||||
$repo->index->setCache($cache, $config['id']);
|
$repo->index->setCache($cache, $config['id']);
|
||||||
}
|
}
|
||||||
$repo->index->load();
|
$repo->index->load();
|
||||||
$repo->loadCatalog($anchor);
|
|
||||||
return $repo;
|
return $repo;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function loadCatalog(?ICatalogAnchor $anchor): void {
|
|
||||||
$this->catalog = new Catalog($this->backend, $this->cipher, $this->repositoryId, $anchor);
|
|
||||||
$this->catalog->load(
|
|
||||||
fn () => array_map('basename', $this->backend->listFiles('snapshots')),
|
|
||||||
fn () => array_map('basename', $this->backend->listFiles('trash/info')),
|
|
||||||
fn () => $this->backend->list('users'));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Refuse uploads that would make the stored data larger than this (null = no limit). */
|
|
||||||
public function setMaxBytes(?int $maxBytes): void {
|
|
||||||
$this->maxBytes = $maxBytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function storedBytes(): int {
|
|
||||||
return $this->index->storedBytes();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Current catalog generation (for status and tests). */
|
|
||||||
public function catalogGeneration(): int {
|
|
||||||
return $this->catalog->gen;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
|
/** Index files downloaded when this repository was opened (0 when the local cache was complete). */
|
||||||
public function indexDownloads(): int {
|
public function indexDownloads(): int {
|
||||||
return $this->index->downloaded;
|
return $this->index->downloaded;
|
||||||
@@ -135,22 +101,14 @@ final class Repository {
|
|||||||
return $config;
|
return $config;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Replace the passphrase-wrapped key slots in this repository's config (after a slot change). */
|
|
||||||
public function updateWrappedKey(array $wrappedKey): void {
|
|
||||||
$config = self::readConfig($this->backend);
|
|
||||||
$config['key'] = $wrappedKey;
|
|
||||||
self::putString($this->backend, 'config', json_encode($config, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR));
|
|
||||||
}
|
|
||||||
|
|
||||||
public function id(): string {
|
public function id(): string {
|
||||||
return $this->repositoryId;
|
return $this->repositoryId;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
public static function open(IBackend $backend, #[\SensitiveParameter] string $passphrase): self {
|
||||||
$config = self::readConfig($backend);
|
$config = self::readConfig($backend);
|
||||||
$repo = new self($backend, KeyRing::unwrapAny($config['key'], $passphrase), $config['id']);
|
$repo = new self($backend, KeyRing::unwrap($config['key'], $passphrase), $config['id']);
|
||||||
$repo->index->load();
|
$repo->index->load();
|
||||||
$repo->loadCatalog(null); // disaster recovery on a new server: no anchor yet, trust the newest valid chain
|
|
||||||
return $repo;
|
return $repo;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -163,7 +121,7 @@ final class Repository {
|
|||||||
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
public function backupDirectory(string $source, ?string $parent = null, string $label = ''): array {
|
||||||
$source = rtrim($source, '/');
|
$source = rtrim($source, '/');
|
||||||
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
$previous = $parent !== null ? $this->loadEntries($parent) : [];
|
||||||
$packs = new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
$packs = new PackWriter($this->backend, $this->cipher, $this->index);
|
||||||
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
$stats = ['files' => 0, 'reused' => 0, 'read' => 0, 'newBlobs' => 0, 'dupBlobs' => 0, 'uploaded' => 0];
|
||||||
|
|
||||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
@@ -191,65 +149,35 @@ final class Repository {
|
|||||||
rewind($encrypted);
|
rewind($encrypted);
|
||||||
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
||||||
fclose($tree);
|
fclose($tree);
|
||||||
$this->catalog->snapshots[$snapshotId] = true;
|
|
||||||
$this->catalog->write();
|
|
||||||
|
|
||||||
return ['snapshot' => $snapshotId] + $stats;
|
return ['snapshot' => $snapshotId] + $stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** Restore a snapshot (or only paths starting with $prefix) into a local directory. */
|
||||||
* Restore a snapshot (or only paths starting with $prefix) into a local directory.
|
public function restore(string $snapshotId, string $target, string $prefix = ''): int {
|
||||||
*
|
|
||||||
* @param bool $stripPrefix write "$target/<path without $prefix>" instead of
|
|
||||||
* "$target/<full logical path>" (disaster recovery: restoring e.g. "data/" or
|
|
||||||
* "config/" directly into the real data/config directory, not a subdirectory
|
|
||||||
* named after the prefix). Ignored when $prefix is '' (nothing to strip).
|
|
||||||
*/
|
|
||||||
public function restore(string $snapshotId, string $target, string $prefix = '', bool $stripPrefix = false, ?callable $heartbeat = null): int {
|
|
||||||
if ($prefix !== '' && !self::isSafePath(rtrim($prefix, '/'))) {
|
|
||||||
throw new RepositoryException('Invalid path prefix');
|
|
||||||
}
|
|
||||||
$count = 0;
|
$count = 0;
|
||||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
foreach ($this->streamEntries($snapshotId) as $entry) {
|
||||||
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
if ($prefix !== '' && !str_starts_with($entry['p'], $prefix)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
$relPath = ($stripPrefix && $prefix !== '') ? substr($entry['p'], strlen(rtrim($prefix, '/')) + 1) : $entry['p'];
|
$dest = rtrim($target, '/') . '/' . $entry['p'];
|
||||||
$dest = rtrim($target, '/') . '/' . $relPath;
|
|
||||||
if (!is_dir(dirname($dest))) {
|
if (!is_dir(dirname($dest))) {
|
||||||
mkdir(dirname($dest), 0700, true);
|
mkdir(dirname($dest), 0700, true);
|
||||||
}
|
}
|
||||||
// Defence in depth: the resolved directory must stay below the target (no symlink escapes).
|
|
||||||
$base = realpath($target);
|
|
||||||
$dir = realpath(dirname($dest));
|
|
||||||
if ($base === false || $dir === false || ($dir !== $base && !str_starts_with($dir, $base . '/')) || is_link($dest)) {
|
|
||||||
throw new RepositoryException('Refusing to write outside the restore directory: ' . $entry['p']);
|
|
||||||
}
|
|
||||||
// Write next to the destination and rename after all blobs verified: at most one file
|
// Write next to the destination and rename after all blobs verified: at most one file
|
||||||
// is ever present twice.
|
// is ever present twice.
|
||||||
$tmp = $dest . '.ngb-restore-' . bin2hex(random_bytes(6));
|
$tmp = $dest . '.ngb-restore';
|
||||||
$out = @fopen($tmp, 'xb'); // fails if anything (including a symlink) already exists there
|
$out = fopen($tmp, 'wb');
|
||||||
if ($out === false) {
|
|
||||||
throw new RepositoryException('Cannot create a temporary file for ' . $entry['p']);
|
|
||||||
}
|
|
||||||
foreach ($entry['b'] as $id) {
|
foreach ($entry['b'] as $id) {
|
||||||
fwrite($out, $this->loadBlob($id));
|
fwrite($out, $this->loadBlob($id));
|
||||||
if ($heartbeat !== null) {
|
|
||||||
$heartbeat();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
fclose($out);
|
fclose($out);
|
||||||
if (filesize($tmp) !== $entry['s']) {
|
if (filesize($tmp) !== $entry['s']) {
|
||||||
unlink($tmp);
|
unlink($tmp);
|
||||||
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
throw new RepositoryException('Size mismatch restoring ' . $entry['p']);
|
||||||
}
|
}
|
||||||
if (!rename($tmp, $dest)) {
|
rename($tmp, $dest);
|
||||||
@unlink($tmp);
|
touch($dest, $entry['m']);
|
||||||
throw new RepositoryException('Cannot put the restored file in place: ' . $entry['p']);
|
|
||||||
}
|
|
||||||
if (!touch($dest, $entry['m'])) {
|
|
||||||
throw new RepositoryException('Cannot set the modification time of ' . $entry['p']);
|
|
||||||
}
|
|
||||||
$count++;
|
$count++;
|
||||||
}
|
}
|
||||||
return $count;
|
return $count;
|
||||||
@@ -260,22 +188,13 @@ final class Repository {
|
|||||||
* run is done, so the last pack is uploaded before anything references it.
|
* run is done, so the last pack is uploaded before anything references it.
|
||||||
*/
|
*/
|
||||||
public function blobWriter(array &$stats): BlobWriter {
|
public function blobWriter(array &$stats): BlobWriter {
|
||||||
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
||||||
$packs = $this->runPacks;
|
$packs = $this->runPacks;
|
||||||
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
|
return new BlobWriter(function (string $data) use ($packs, &$stats): string {
|
||||||
return $this->storeBlob($data, $packs, $stats);
|
return $this->storeBlob($data, $packs, $stats);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/** See BlobIndex::holdCache(): no database writes until releaseIndexCache(). */
|
|
||||||
public function holdIndexCache(): void {
|
|
||||||
$this->index->holdCache();
|
|
||||||
}
|
|
||||||
|
|
||||||
public function releaseIndexCache(): void {
|
|
||||||
$this->index->releaseCache();
|
|
||||||
}
|
|
||||||
|
|
||||||
public function flushPacks(): void {
|
public function flushPacks(): void {
|
||||||
$this->runPacks?->flush();
|
$this->runPacks?->flush();
|
||||||
}
|
}
|
||||||
@@ -301,7 +220,7 @@ final class Repository {
|
|||||||
|
|
||||||
/** Store one blob of file data in the current run (deduplicated); returns its id. */
|
/** Store one blob of file data in the current run (deduplicated); returns its id. */
|
||||||
public function storeData(string $data, array &$stats): string {
|
public function storeData(string $data, array &$stats): string {
|
||||||
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index, 32 * 1048576, $this->maxBytes);
|
$this->runPacks ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
||||||
return $this->storeBlob($data, $this->runPacks, $stats);
|
return $this->storeBlob($data, $this->runPacks, $stats);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -312,17 +231,6 @@ final class Repository {
|
|||||||
* @param iterable<string> $lines
|
* @param iterable<string> $lines
|
||||||
*/
|
*/
|
||||||
public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
public function writeSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
||||||
$this->putSnapshot($snapshotId, $meta, $lines);
|
|
||||||
$this->recordSnapshot($snapshotId);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Store a snapshot object without touching the catalog. Use recordSnapshot() afterwards, from a
|
|
||||||
* repository opened inside the catalog lock, so the catalog write builds on the latest generation.
|
|
||||||
*
|
|
||||||
* @param iterable<string> $lines
|
|
||||||
*/
|
|
||||||
public function putSnapshot(string $snapshotId, array $meta, iterable $lines): void {
|
|
||||||
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$tree = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n");
|
fwrite($tree, json_encode(['snapshot' => $snapshotId] + $meta, JSON_THROW_ON_ERROR) . "\n");
|
||||||
foreach ($lines as $line) {
|
foreach ($lines as $line) {
|
||||||
@@ -338,11 +246,6 @@ final class Repository {
|
|||||||
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function recordSnapshot(string $snapshotId): void {
|
|
||||||
$this->catalog->snapshots[$snapshotId] = true;
|
|
||||||
$this->catalog->write();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
/** @return array<string, array{s:int, m:int, b:list<string>}> */
|
||||||
public function snapshotEntries(string $snapshotId): array {
|
public function snapshotEntries(string $snapshotId): array {
|
||||||
return $this->loadEntries($snapshotId);
|
return $this->loadEntries($snapshotId);
|
||||||
@@ -358,19 +261,6 @@ final class Repository {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Relative path of non-empty segments, without ".", "..", NUL, backslash or control characters. */
|
|
||||||
public static function isSafePath(string $path): bool {
|
|
||||||
if ($path === '' || str_starts_with($path, '/') || preg_match('/[\x00-\x1f\\\\]/', $path)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
foreach (explode('/', $path) as $segment) {
|
|
||||||
if ($segment === '' || $segment === '.' || $segment === '..') {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Small encrypted objects (manifests). */
|
/** Small encrypted objects (manifests). */
|
||||||
public function putObject(string $path, string $data): void {
|
public function putObject(string $path, string $data): void {
|
||||||
self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path));
|
self::putString($this->backend, $path, $this->cipher->encryptString($data, 'object:' . $path));
|
||||||
@@ -383,280 +273,9 @@ final class Repository {
|
|||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** @return list<string> */
|
||||||
* Move a snapshot (and its database dump) to the trash on the location. Its data stays in
|
|
||||||
* use until purgeTrash() removes it after the deletion delay, so a malicious or mistaken
|
|
||||||
* cleanup can be undone with untrash() within that time.
|
|
||||||
*/
|
|
||||||
public function forget(string $snapshotId, string $by = ''): void {
|
|
||||||
$meta = $this->snapshotMeta($snapshotId);
|
|
||||||
$dbInTrash = null;
|
|
||||||
if (isset($meta['db']) && is_string($meta['db']) && $this->backend->exists($meta['db'])) {
|
|
||||||
// Its encryption is bound to its path (object:db/...), which changes on a move, so it
|
|
||||||
// must be decrypted and re-encrypted under the new path, not moved as raw bytes.
|
|
||||||
$dbInTrash = 'trash/' . $meta['db'];
|
|
||||||
$this->putObject($dbInTrash, $this->getObject($meta['db']));
|
|
||||||
}
|
|
||||||
// Copy first, switch the catalog, delete the originals last: a crash at any point leaves
|
|
||||||
// either the old state or the new one plus unreferenced leftovers, never a catalog entry
|
|
||||||
// pointing at a missing object. The snapshot object is bound to its id (not its path),
|
|
||||||
// so it can be copied as raw bytes.
|
|
||||||
$this->copyRaw('snapshots/' . $snapshotId, 'trash/snapshots/' . $snapshotId);
|
|
||||||
$this->putObject('trash/info/' . $snapshotId, json_encode(['forgottenAt' => time(), 'by' => $by, 'db' => $meta['db'] ?? null,
|
|
||||||
'dbInTrash' => $dbInTrash, 'time' => $meta['time'] ?? null, 'label' => $meta['label'] ?? ''], JSON_THROW_ON_ERROR));
|
|
||||||
unset($this->catalog->snapshots[$snapshotId]);
|
|
||||||
$this->catalog->trash[$snapshotId] = true;
|
|
||||||
$this->catalog->write();
|
|
||||||
if ($dbInTrash !== null) {
|
|
||||||
$this->backend->delete($meta['db']);
|
|
||||||
}
|
|
||||||
$this->backend->delete('snapshots/' . $snapshotId);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Undo forget(): move a snapshot back from the trash. */
|
|
||||||
public function untrash(string $snapshotId): void {
|
|
||||||
$info = $this->trashInfo($snapshotId);
|
|
||||||
$restoreDb = $info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash']);
|
|
||||||
if ($restoreDb) {
|
|
||||||
$this->putObject($info['db'], $this->getObject($info['dbInTrash']));
|
|
||||||
}
|
|
||||||
// Same order as forget(): copy, switch the catalog, then delete the trash copies.
|
|
||||||
$this->copyRaw('trash/snapshots/' . $snapshotId, 'snapshots/' . $snapshotId);
|
|
||||||
$this->catalog->snapshots[$snapshotId] = true;
|
|
||||||
unset($this->catalog->trash[$snapshotId]);
|
|
||||||
$this->catalog->write();
|
|
||||||
if ($restoreDb) {
|
|
||||||
$this->backend->delete($info['dbInTrash']);
|
|
||||||
}
|
|
||||||
$this->backend->delete('trash/snapshots/' . $snapshotId);
|
|
||||||
$this->backend->delete('trash/info/' . $snapshotId);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Copy an object as raw bytes, buffered locally (remote read streams cannot be uploaded directly everywhere). */
|
|
||||||
private function copyRaw(string $from, string $to): void {
|
|
||||||
$in = $this->backend->get($from);
|
|
||||||
$buffer = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
|
||||||
if ($buffer === false) {
|
|
||||||
fclose($in);
|
|
||||||
throw new RepositoryException("Cannot buffer $from");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
if (stream_copy_to_stream($in, $buffer) === false) {
|
|
||||||
throw new RepositoryException("Cannot read $from");
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
fclose($in);
|
|
||||||
}
|
|
||||||
rewind($buffer);
|
|
||||||
$this->backend->put($to, $buffer);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Record a user_migration export's manifest in the catalog (rollback-protected like a snapshot). */
|
|
||||||
public function recordUserExport(string $manifestPath): void {
|
|
||||||
$this->catalog->userExports[$manifestPath] = true;
|
|
||||||
$this->catalog->write();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* User export manifests according to the verified catalog, newest first. A manifest the
|
|
||||||
* catalog lists but the location no longer has means it was deleted or hidden behind NG
|
|
||||||
* Backup's back, same as a missing snapshot (see listSnapshots()).
|
|
||||||
*
|
|
||||||
* @return list<string>
|
|
||||||
*/
|
|
||||||
public function listUserExports(?string $uid = null): array {
|
|
||||||
$prefix = $uid !== null ? 'users/' . $uid . '/' : 'users/';
|
|
||||||
$out = [];
|
|
||||||
foreach (array_keys($this->catalog->userExports) as $path) {
|
|
||||||
if (!str_starts_with($path, $prefix)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (!$this->backend->exists($path)) {
|
|
||||||
throw new RepositoryException("User export $path is missing on the location (deleted or hidden outside NG Backup)");
|
|
||||||
}
|
|
||||||
$out[] = $path;
|
|
||||||
}
|
|
||||||
sort($out, SORT_STRING);
|
|
||||||
return array_reverse($out);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Whether $manifestPath is a user export the verified catalog actually recorded. */
|
|
||||||
public function isRecordedUserExport(string $manifestPath): bool {
|
|
||||||
return isset($this->catalog->userExports[$manifestPath]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Permanently delete a user export. Its blobs are freed by the next prune() like any other. */
|
|
||||||
public function forgetUserExport(string $manifestPath): void {
|
|
||||||
if (!isset($this->catalog->userExports[$manifestPath])) {
|
|
||||||
throw new RepositoryException("$manifestPath is not a recorded user export");
|
|
||||||
}
|
|
||||||
unset($this->catalog->userExports[$manifestPath]);
|
|
||||||
$this->catalog->write();
|
|
||||||
$this->backend->delete($manifestPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return array<string, array{forgottenAt:int, by:string, db:?string, dbInTrash:?string, time:?string, label:string}> */
|
|
||||||
public function trash(): array {
|
|
||||||
$out = [];
|
|
||||||
foreach (array_keys($this->catalog->trash) as $id) {
|
|
||||||
try {
|
|
||||||
$out[$id] = $this->trashInfo($id);
|
|
||||||
} catch (\Throwable) {
|
|
||||||
throw new RepositoryException("Trash entry $id listed in the catalog is missing on the location (possible tampering)");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return $out;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Permanently delete trash entries older than $delaySeconds. @return list<string> purged snapshot ids */
|
|
||||||
public function purgeTrash(int $delaySeconds): array {
|
|
||||||
$doomed = [];
|
|
||||||
foreach ($this->trash() as $id => $info) {
|
|
||||||
if (time() - $info['forgottenAt'] < $delaySeconds) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$doomed[$id] = $info;
|
|
||||||
unset($this->catalog->trash[$id]);
|
|
||||||
}
|
|
||||||
if ($doomed === []) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
$this->catalog->write();
|
|
||||||
foreach ($doomed as $id => $info) {
|
|
||||||
if ($info['dbInTrash'] !== null) {
|
|
||||||
$this->backend->delete($info['dbInTrash']);
|
|
||||||
}
|
|
||||||
$this->backend->delete('trash/snapshots/' . $id);
|
|
||||||
$this->backend->delete('trash/info/' . $id);
|
|
||||||
}
|
|
||||||
return array_keys($doomed);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function trashInfo(string $snapshotId): array {
|
|
||||||
return json_decode($this->getObject('trash/info/' . $snapshotId), true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete packs no snapshot or user export uses any more, and repack packs where more than
|
|
||||||
* $repackThreshold of the bytes are unused. Must not run while a backup to this repository
|
|
||||||
* is in progress (its blobs are not referenced by a snapshot yet); callers hold the lock.
|
|
||||||
*
|
|
||||||
* @return array{packs:int, deleted:int, repacked:int, freedBytes:int, keptBlobs:int, unusedBlobs:int}
|
|
||||||
*/
|
|
||||||
public function prune(bool $dryRun = false, float $repackThreshold = 0.5, array $ignoreSnapshots = []): array {
|
|
||||||
// Deleting is decided on the authenticated index files on the location, never on the local cache.
|
|
||||||
$this->index->load(true);
|
|
||||||
$used = [];
|
|
||||||
$ignore = array_flip($ignoreSnapshots); // e.g. snapshots a dry run would forget first
|
|
||||||
foreach ($this->listSnapshots() as $sid) {
|
|
||||||
if (isset($ignore[$sid])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
foreach ($this->streamEntries($sid) as $e) {
|
|
||||||
foreach ($e['b'] as $b) {
|
|
||||||
$used[$b] = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$meta = $this->snapshotMeta($sid);
|
|
||||||
if (isset($meta['db']) && $this->backend->exists($meta['db'])) {
|
|
||||||
foreach (json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
|
||||||
foreach ($t['blobs'] as $b) {
|
|
||||||
$used[$b] = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Snapshots in the trash keep their data until purgeTrash() removed them.
|
|
||||||
foreach ($this->trash() as $sid => $info) {
|
|
||||||
foreach ($this->streamEntries($sid, 'trash/snapshots/') as $e) {
|
|
||||||
foreach ($e['b'] as $b) {
|
|
||||||
$used[$b] = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($info['dbInTrash'] !== null && $this->backend->exists($info['dbInTrash'])) {
|
|
||||||
foreach (json_decode($this->getObject($info['dbInTrash']), true, 512, JSON_THROW_ON_ERROR)['tables'] as $t) {
|
|
||||||
foreach ($t['blobs'] as $b) {
|
|
||||||
$used[$b] = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// listUserExports() throws when a recorded export's manifest is missing: then stop, since
|
|
||||||
// freeing its blobs would turn a tampering or storage alarm into irreversible deletion.
|
|
||||||
$exports = array_unique(array_merge($this->listUserExports(), $this->backend->list('users')));
|
|
||||||
foreach ($exports as $path) {
|
|
||||||
foreach (json_decode($this->getObject($path), true, 512, JSON_THROW_ON_ERROR)['entries'] as $e) {
|
|
||||||
foreach ($e['b'] ?? [] as $b) {
|
|
||||||
$used[$b] = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$stats = ['packs' => 0, 'deleted' => 0, 'repacked' => 0, 'freedBytes' => 0, 'keptBlobs' => count($used), 'unusedBlobs' => 0];
|
|
||||||
$packs = $this->index->packs();
|
|
||||||
$stats['packs'] = count($packs);
|
|
||||||
$writer = null;
|
|
||||||
$toRemove = [];
|
|
||||||
foreach ($packs as $packId => $blobs) {
|
|
||||||
$total = 0;
|
|
||||||
$unused = 0;
|
|
||||||
foreach ($blobs as $b) {
|
|
||||||
$total += $b['length'];
|
|
||||||
if (!isset($used[$b['id']])) {
|
|
||||||
$unused += $b['length'];
|
|
||||||
$stats['unusedBlobs']++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($unused === 0 || ($unused < $total && $unused / $total <= $repackThreshold)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$stats['freedBytes'] += $unused;
|
|
||||||
if ($unused === $total) {
|
|
||||||
$stats['deleted']++;
|
|
||||||
} else {
|
|
||||||
$stats['repacked']++;
|
|
||||||
if (!$dryRun) {
|
|
||||||
// No size limit here: repacking is how space is freed.
|
|
||||||
$writer ??= new PackWriter($this->backend, $this->cipher, $this->index);
|
|
||||||
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
|
||||||
foreach ($blobs as $b) {
|
|
||||||
if (isset($used[$b['id']])) {
|
|
||||||
[, , , $raw, $flags] = $this->index->get($b['id']);
|
|
||||||
$writer->addRaw($b['id'], $this->backend->getRange($path, $b['offset'], $b['length']), $raw, $flags);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$toRemove[] = $packId;
|
|
||||||
}
|
|
||||||
if ($dryRun) {
|
|
||||||
return $stats;
|
|
||||||
}
|
|
||||||
// New packs and their index entries are durable before any old pack is removed.
|
|
||||||
$writer?->flush();
|
|
||||||
foreach ($toRemove as $packId) {
|
|
||||||
$this->index->removePack($packId);
|
|
||||||
$this->backend->delete('packs/' . substr($packId, 0, 2) . '/' . $packId);
|
|
||||||
}
|
|
||||||
return $stats;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Snapshots according to the verified catalog. A snapshot the catalog lists but the location
|
|
||||||
* no longer has means it was deleted or hidden behind NG Backup's back.
|
|
||||||
*
|
|
||||||
* @return list<string>
|
|
||||||
*/
|
|
||||||
public function listSnapshots(): array {
|
public function listSnapshots(): array {
|
||||||
$present = array_flip(array_map('basename', $this->backend->listFiles('snapshots')));
|
return array_map('basename', $this->backend->list('snapshots'));
|
||||||
$listed = array_keys($this->catalog->snapshots);
|
|
||||||
foreach ($listed as $id) {
|
|
||||||
if (!isset($present[$id])) {
|
|
||||||
throw new RepositoryException("Snapshot $id is missing on the location (deleted or hidden outside NG Backup)");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort($listed);
|
|
||||||
return $listed;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function blobCount(): int {
|
public function blobCount(): int {
|
||||||
@@ -712,13 +331,8 @@ final class Repository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private function loadBlob(string $id): string {
|
private function loadBlob(string $id): string {
|
||||||
[$pack, $offset, $length] = $this->index->get($id);
|
[$pack, $offset, $length, $raw, $flags] = $this->index->get($id);
|
||||||
return $this->decodeBlob($id, $this->packReader->read($pack, $offset, $length));
|
$encrypted = $this->backend->getRange('packs/' . substr($pack, 0, 2) . '/' . $pack, $offset, $length);
|
||||||
}
|
|
||||||
|
|
||||||
/** Decrypts and checks a blob read from its pack; throws when it is corrupted or tampered with. */
|
|
||||||
private function decodeBlob(string $id, string $encrypted): string {
|
|
||||||
[, , , $raw, $flags] = $this->index->get($id);
|
|
||||||
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
$payload = $this->cipher->decryptString($encrypted, 'blob:' . $id);
|
||||||
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
$data = ($flags & self::FLAG_DEFLATE) ? gzinflate($payload) : $payload;
|
||||||
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
if ($data === false || strlen($data) !== $raw || $this->keys->blobId($data) !== $id) {
|
||||||
@@ -741,184 +355,6 @@ final class Repository {
|
|||||||
return $this->streamEntries($snapshotId);
|
return $this->streamEntries($snapshotId);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Checksum audit of a snapshot: every blob it references (files and the database dump, when
|
|
||||||
* the snapshot has one) must be in the index, freshly re-downloaded and re-authenticated from
|
|
||||||
* the location rather than trusted from the local cache (same reasoning as prune()), and its
|
|
||||||
* pack must still be present. With $deep, each blob is also downloaded and decrypted, which
|
|
||||||
* verifies its AEAD authentication tag and its content hash against the blob id (loadBlob()
|
|
||||||
* already does both on every real read -- this just does it for everything in the snapshot,
|
|
||||||
* deliberately, rather than relying on it to come up during a later restore). A blob
|
|
||||||
* referenced by more than one file is only checked once; a missing database manifest is a
|
|
||||||
* failure, not something to silently skip over.
|
|
||||||
*
|
|
||||||
* @param ?callable(array{phase:string, done:int, total:?int, blobs?:int, bytes?:int}):void $heartbeat
|
|
||||||
* called often during the check, with how far it is: phase "index" (index files
|
|
||||||
* downloaded), "packs" (pack folders listed), "files" and "database" (files of the
|
|
||||||
* snapshot checked); callers use it to keep a time-boxed lease alive and to report
|
|
||||||
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
|
||||||
*/
|
|
||||||
public function verify(string $snapshotId, bool $deep = false, ?callable $heartbeat = null): array {
|
|
||||||
$heartbeat ??= static function (array $progress): void {
|
|
||||||
};
|
|
||||||
$meta = $this->snapshotMeta($snapshotId);
|
|
||||||
$filesTotal = isset($meta['stats']['files']) ? (int)$meta['stats']['files'] : null;
|
|
||||||
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
|
|
||||||
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
|
|
||||||
});
|
|
||||||
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
|
||||||
$seen = [];
|
|
||||||
[$packSeen, $listedDirs] = $this->presentPacks($heartbeat);
|
|
||||||
foreach ($this->streamEntries($snapshotId) as $entry) {
|
|
||||||
$stats['filesChecked']++;
|
|
||||||
foreach ($entry['b'] as $id) {
|
|
||||||
if (isset($seen[$id])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$seen[$id] = true;
|
|
||||||
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
|
|
||||||
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
|
|
||||||
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (isset($meta['db']) && is_string($meta['db'])) {
|
|
||||||
if (!$this->backend->exists($meta['db'])) {
|
|
||||||
$stats['missing'][] = $meta['db'];
|
|
||||||
} else {
|
|
||||||
$dbManifest = json_decode($this->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
foreach ($dbManifest['tables'] as $info) {
|
|
||||||
foreach ($info['blobs'] as $id) {
|
|
||||||
if (isset($seen[$id])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$seen[$id] = true;
|
|
||||||
$this->verifyBlob($id, $deep, $stats, $packSeen, $listedDirs);
|
|
||||||
$heartbeat(['phase' => 'database', 'done' => $stats['filesChecked'], 'total' => $filesTotal,
|
|
||||||
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return $stats;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Which packs are on the location, from one listing per pack folder instead of a request per
|
|
||||||
* pack. A folder that cannot be listed, or comes back empty (which a storage also reports when
|
|
||||||
* it could not look), is left out: its packs are then checked one by one. With no more packs
|
|
||||||
* than folders nothing is listed: a request per pack is fewer requests then, and on FTP every
|
|
||||||
* request takes a data connection, of which a server may have only a few.
|
|
||||||
*
|
|
||||||
* @return array{0: array<string, bool>, 1: array<string, true>} pack id => present, and the
|
|
||||||
* folders that were listed
|
|
||||||
*/
|
|
||||||
private function presentPacks(callable $heartbeat): array {
|
|
||||||
$present = [];
|
|
||||||
$listed = [];
|
|
||||||
if ($this->index->packCount() <= $this->listPacksAbove) {
|
|
||||||
return [$present, $listed];
|
|
||||||
}
|
|
||||||
for ($i = 0; $i < 256; $i++) {
|
|
||||||
$dir = sprintf('%02x', $i);
|
|
||||||
try {
|
|
||||||
$files = $this->backend->listFiles('packs/' . $dir);
|
|
||||||
} catch (BackendException) {
|
|
||||||
$files = [];
|
|
||||||
}
|
|
||||||
foreach ($files as $path) {
|
|
||||||
$present[basename($path)] = true;
|
|
||||||
}
|
|
||||||
if ($files !== []) {
|
|
||||||
$listed[$dir] = true;
|
|
||||||
}
|
|
||||||
$heartbeat(['phase' => 'packs', 'done' => $i + 1, 'total' => 256]);
|
|
||||||
}
|
|
||||||
return [$present, $listed];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Same checks as verify(), for a user export: every blob its manifest references must be in
|
|
||||||
* the freshly re-authenticated index with its pack present; $deep also decrypts each.
|
|
||||||
*
|
|
||||||
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
|
||||||
*/
|
|
||||||
public function verifyUserExport(string $manifestPath, bool $deep = false, ?callable $heartbeat = null): array {
|
|
||||||
if (!$this->isRecordedUserExport($manifestPath)) {
|
|
||||||
throw new RepositoryException("$manifestPath is not a recorded user export for this location");
|
|
||||||
}
|
|
||||||
$heartbeat ??= static function (array $progress): void {
|
|
||||||
};
|
|
||||||
$this->index->load(true, static function (int $done, int $total) use ($heartbeat): void {
|
|
||||||
$heartbeat(['phase' => 'index', 'done' => $done, 'total' => $total]);
|
|
||||||
});
|
|
||||||
$stats = ['filesChecked' => 0, 'blobsChecked' => 0, 'bytesChecked' => 0, 'missing' => [], 'failed' => [], 'unreadable' => [], 'readError' => ''];
|
|
||||||
$seen = [];
|
|
||||||
$packSeen = [];
|
|
||||||
$manifest = json_decode($this->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
foreach ($manifest['entries'] as $entry) {
|
|
||||||
if (($entry['t'] ?? '') !== 'f') {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$stats['filesChecked']++;
|
|
||||||
foreach ($entry['b'] ?? [] as $id) {
|
|
||||||
if (isset($seen[$id])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$seen[$id] = true;
|
|
||||||
$this->verifyBlob($id, $deep, $stats, $packSeen);
|
|
||||||
$heartbeat(['phase' => 'files', 'done' => $stats['filesChecked'], 'total' => null,
|
|
||||||
'blobs' => $stats['blobsChecked'], 'bytes' => $stats['bytesChecked']]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return $stats;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param array<string, bool> $packSeen pack id => present; filled in as packs are checked
|
|
||||||
* @param array<string, true> $listedDirs pack folders whose listing is in $packSeen: a pack
|
|
||||||
* not in it is not there
|
|
||||||
*/
|
|
||||||
private function verifyBlob(string $id, bool $deep, array &$stats, array &$packSeen, array $listedDirs = []): void {
|
|
||||||
if (!$this->index->has($id)) {
|
|
||||||
$stats['missing'][] = $id;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
[$pack] = $this->index->get($id);
|
|
||||||
$packExists = $packSeen[$pack] ??= isset($listedDirs[substr($pack, 0, 2)])
|
|
||||||
? false
|
|
||||||
: $this->backend->exists('packs/' . substr($pack, 0, 2) . '/' . $pack);
|
|
||||||
if (!$packExists) {
|
|
||||||
$stats['missing'][] = $id;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$stats['blobsChecked']++;
|
|
||||||
if (!$deep) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Reading and checking are kept apart: a read that fails is usually a network or
|
|
||||||
// storage error, so it is retried and reported as unreadable (a pack that stays
|
|
||||||
// unreadable may still be truncated). Only a blob that was read but does not
|
|
||||||
// decrypt or match its id counts as failed: that is proof of damage.
|
|
||||||
[, $offset, $length] = $this->index->get($id);
|
|
||||||
$encrypted = null;
|
|
||||||
for ($attempt = 1; $attempt <= self::VERIFY_READ_ATTEMPTS && $encrypted === null; $attempt++) {
|
|
||||||
try {
|
|
||||||
$encrypted = $this->packReader->read($pack, $offset, $length);
|
|
||||||
} catch (\Exception $e) { // storage adapters throw their own exceptions; \Error (a bug) is not retried
|
|
||||||
$stats['readError'] = $e->getMessage();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($encrypted === null) {
|
|
||||||
$stats['unreadable'][] = $id;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$stats['bytesChecked'] += strlen($this->decodeBlob($id, $encrypted));
|
|
||||||
} catch (\Exception) { // damage; an \Error is a bug and must not be reported as damage
|
|
||||||
$stats['failed'][] = $id;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The meta record (first line) of a snapshot. */
|
/** The meta record (first line) of a snapshot. */
|
||||||
public function snapshotMeta(string $snapshotId): array {
|
public function snapshotMeta(string $snapshotId): array {
|
||||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
@@ -932,20 +368,15 @@ final class Repository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
/** @return \Generator<array{p:string, s:int, m:int, b:list<string>}> */
|
||||||
private function streamEntries(string $snapshotId, string $dir = 'snapshots/'): \Generator {
|
private function streamEntries(string $snapshotId): \Generator {
|
||||||
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
$plain = fopen('php://temp/maxmemory:' . (4 * 1048576), 'w+b');
|
||||||
$fh = $this->backend->get($dir . $snapshotId);
|
$fh = $this->backend->get('snapshots/' . $snapshotId);
|
||||||
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
$this->cipher->decrypt($fh, $plain, 'snapshot:' . $snapshotId);
|
||||||
fclose($fh);
|
fclose($fh);
|
||||||
rewind($plain);
|
rewind($plain);
|
||||||
fgets($plain); // meta line
|
fgets($plain); // meta line
|
||||||
while (($line = fgets($plain)) !== false) {
|
while (($line = fgets($plain)) !== false) {
|
||||||
$entry = json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
yield json_decode($line, true, 512, JSON_THROW_ON_ERROR);
|
||||||
if (!is_array($entry) || !is_string($entry['p'] ?? null) || !self::isSafePath($entry['p'])
|
|
||||||
|| !is_int($entry['s'] ?? null) || !is_array($entry['b'] ?? null)) {
|
|
||||||
throw new RepositoryException('Invalid entry in snapshot ' . $snapshotId);
|
|
||||||
}
|
|
||||||
yield $entry;
|
|
||||||
}
|
}
|
||||||
fclose($plain);
|
fclose($plain);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,121 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
use OCA\NgBackup\AppInfo\Application;
|
|
||||||
use OCA\NgBackup\Db\Run;
|
|
||||||
use OCA\NgBackup\Db\RunMapper;
|
|
||||||
use OCA\NgBackup\Db\Target;
|
|
||||||
use OCP\IGroupManager;
|
|
||||||
use OCP\Notification\IManager;
|
|
||||||
|
|
||||||
/** Notifies administrators about failed runs and locations without a recent successful backup. */
|
|
||||||
final class AlertService {
|
|
||||||
public const STALE_AFTER = 2 * 86400;
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private IManager $notifications,
|
|
||||||
private IGroupManager $groups,
|
|
||||||
private RunMapper $runs,
|
|
||||||
private \OCP\IAppConfig $appConfig,
|
|
||||||
private \OCP\EventDispatcher\IEventDispatcher $events,
|
|
||||||
private \OCP\IUserSession $session,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Security-relevant change (recovery kit downloaded, location removed, append-only off,
|
|
||||||
* retention changed, snapshots forgotten): written to the audit log (admin_audit) and sent
|
|
||||||
* to every administrator, so a single compromised admin account cannot do it unnoticed.
|
|
||||||
*
|
|
||||||
* @param array<string, string> $params
|
|
||||||
*/
|
|
||||||
public function actor(): string {
|
|
||||||
return $this->session->getUser()?->getUID() ?? (PHP_SAPI === 'cli' ? 'occ' : 'unknown');
|
|
||||||
}
|
|
||||||
|
|
||||||
public function securityEvent(string $event, array $params = [], bool $notify = true): void {
|
|
||||||
$actor = $this->actor();
|
|
||||||
$params = ['actor' => $actor] + $params;
|
|
||||||
$this->events->dispatchTyped(new \OCP\Log\Audit\CriticalActionPerformedEvent(
|
|
||||||
'NG Backup: %s by %s ' . json_encode(array_diff_key($params, ['actor' => 1]), JSON_UNESCAPED_SLASHES), ['event' => $event, 'actor' => $actor]));
|
|
||||||
if ($notify) {
|
|
||||||
$this->notifyAdmins('security_event', $event . '-' . time(), ['event' => $event] + $params);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function runFailed(Run $run, string $targetName): void {
|
|
||||||
$this->notifyAdmins('run_failed', 'run-' . $run->getId(),
|
|
||||||
['kind' => $run->getKind(), 'target' => $targetName, 'error' => mb_strimwidth((string)$run->getError(), 0, 300, '…')]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** A backup that finished but is missing unreadable files, or holds files that changed while read. */
|
|
||||||
public function runWarnings(Run $run, string $targetName, int $unreadable, int $changing, array $examples): void {
|
|
||||||
$this->notifyAdmins('run_warnings', 'run-' . $run->getId(), ['target' => $targetName,
|
|
||||||
'unreadable' => (string)$unreadable, 'changing' => (string)$changing,
|
|
||||||
'examples' => mb_strimwidth(implode(', ', array_merge($examples['unreadable'], $examples['changing'])), 0, 300, '…')]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Remember how much is stored on a location (for the settings page) and warn administrators
|
|
||||||
* once a day when it is at 80% of its limit or more.
|
|
||||||
*/
|
|
||||||
public function storedBytes(Target $target, int $bytes): void {
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, 'stored_' . $target->getId(), (string)$bytes);
|
|
||||||
$limit = $target->getMaxBytes();
|
|
||||||
if ($limit === null || $limit <= 0 || $bytes < 0.8 * $limit) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$key = 'quota_notified_' . $target->getId();
|
|
||||||
if ($this->appConfig->getValueString(Application::APP_ID, $key, '') === date('Y-m-d')) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, $key, date('Y-m-d'));
|
|
||||||
$this->notifyAdmins('quota_warning', 'quota-' . $target->getId() . '-' . date('Y-m-d'), [
|
|
||||||
'target' => $target->getName(), 'percent' => (string)round($bytes / $limit * 100),
|
|
||||||
'used' => self::gib($bytes), 'limit' => self::gib($limit)]);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function gib(int $bytes): string {
|
|
||||||
$g = $bytes / 1073741824;
|
|
||||||
return sprintf($g < 10 ? '%.2f GiB' : '%.1f GiB', $g);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function lastStoredBytes(Target $target): ?int {
|
|
||||||
$v = $this->appConfig->getValueString(Application::APP_ID, 'stored_' . $target->getId(), '');
|
|
||||||
return $v === '' ? null : (int)$v;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** At most one notification per location per day. */
|
|
||||||
public function checkStale(Target $target): void {
|
|
||||||
$last = $this->runs->lastFinished($target->getId(), BackupService::KIND_FULL);
|
|
||||||
$lastTime = $last?->getFinishedAt() ?? $target->getCreatedAt();
|
|
||||||
if (time() - $lastTime < self::STALE_AFTER || $this->runs->findRunning($target->getId(), BackupService::KIND_FULL) !== []) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$key = 'stale_notified_' . $target->getId();
|
|
||||||
if ($this->appConfig->getValueString(Application::APP_ID, $key, '') === date('Y-m-d')) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, $key, date('Y-m-d'));
|
|
||||||
$this->notifyAdmins('backup_stale', 'stale-' . $target->getId() . '-' . date('Y-m-d'),
|
|
||||||
['target' => $target->getName(), 'since' => $last ? date('Y-m-d H:i', $lastTime) : '']);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function notifyAdmins(string $subject, string $objectId, array $params): void {
|
|
||||||
foreach ($this->groups->get('admin')?->getUsers() ?? [] as $admin) {
|
|
||||||
$n = $this->notifications->createNotification();
|
|
||||||
$n->setApp(Application::APP_ID)
|
|
||||||
->setUser($admin->getUID())
|
|
||||||
->setDateTime(new \DateTime())
|
|
||||||
->setObject($subject, $objectId)
|
|
||||||
->setSubject($subject, $params);
|
|
||||||
$this->notifications->notify($n);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+27
-285
@@ -14,12 +14,8 @@ use OCA\NgBackup\Db\Snapshot;
|
|||||||
use OCA\NgBackup\Db\SnapshotMapper;
|
use OCA\NgBackup\Db\SnapshotMapper;
|
||||||
use OCA\NgBackup\Db\Target;
|
use OCA\NgBackup\Db\Target;
|
||||||
use OCA\NgBackup\Job\BackupRun;
|
use OCA\NgBackup\Job\BackupRun;
|
||||||
use OCA\NgBackup\Repository\Repository;
|
|
||||||
use OCA\NgBackup\Repository\RepositoryException;
|
|
||||||
use OCP\AppFramework\Db\DoesNotExistException;
|
|
||||||
use OCP\App\IAppManager;
|
use OCP\App\IAppManager;
|
||||||
use OCP\IConfig;
|
use OCP\IConfig;
|
||||||
use OCP\DB\QueryBuilder\IQueryBuilder;
|
|
||||||
use OCP\IDBConnection;
|
use OCP\IDBConnection;
|
||||||
use OCP\Lock\ILockingProvider;
|
use OCP\Lock\ILockingProvider;
|
||||||
use OCP\Lock\LockedException;
|
use OCP\Lock\LockedException;
|
||||||
@@ -32,12 +28,9 @@ use Psr\Log\LoggerInterface;
|
|||||||
* Excluded by default: previews, log files, the updater's backups and each user's cache and
|
* Excluded by default: previews, log files, the updater's backups and each user's cache and
|
||||||
* temporary uploads (all regenerable or temporary).
|
* temporary uploads (all regenerable or temporary).
|
||||||
*/
|
*/
|
||||||
final class BackupService {
|
class BackupService {
|
||||||
public const KIND_FULL = 'full';
|
public const KIND_FULL = 'full';
|
||||||
|
|
||||||
/** @var \WeakMap<Repository, string> the location settings each kept repository was opened with */
|
|
||||||
private \WeakMap $openedWith;
|
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
@@ -48,76 +41,16 @@ final class BackupService {
|
|||||||
private IAppManager $appManager,
|
private IAppManager $appManager,
|
||||||
private ILockingProvider $locking,
|
private ILockingProvider $locking,
|
||||||
private LoggerInterface $logger,
|
private LoggerInterface $logger,
|
||||||
private AlertService $alerts,
|
|
||||||
private LeaseService $leases,
|
|
||||||
) {
|
) {
|
||||||
/** @var \WeakMap<Repository, string> $opened */
|
|
||||||
$opened = new \WeakMap();
|
|
||||||
$this->openedWith = $opened;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function start(Target $target, string $label = ''): Run {
|
public function start(Target $target, string $label = ''): Run {
|
||||||
if (!$this->keys->everConfirmed()) {
|
if ($this->keys->recoveryKitConfirmation() === null) {
|
||||||
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
|
throw new \RuntimeException('Confirm the recovery kit first (occ backup:key:confirm)');
|
||||||
}
|
}
|
||||||
if ($this->config->getSystemValue('objectstore', null) !== null) {
|
if ($this->config->getSystemValue('objectstore', null) !== null) {
|
||||||
throw new \RuntimeException('Object storage as primary storage is not supported yet');
|
throw new \RuntimeException('Object storage as primary storage is not supported yet');
|
||||||
}
|
}
|
||||||
// Check and create the run under the location's exclusive lock, so a cleanup cannot start
|
|
||||||
// in between (it takes the same lock and refuses while a run exists).
|
|
||||||
// A run that is already in progress is simply continued; no lease needed for that.
|
|
||||||
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
|
||||||
if ($running !== []) {
|
|
||||||
return $running[0];
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $this->createRun($target, $label));
|
|
||||||
} catch (LockedException) {
|
|
||||||
throw new \RuntimeException('A cleanup or another start for ' . $target->getName() . ' is running; try again later');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The expected size of the files phase, for the progress shown while it runs: the size of the
|
|
||||||
* previous snapshot, or for a first backup the users' files, versions and trash as Nextcloud's
|
|
||||||
* file cache knows them (approximate: other data in the data directory is not counted).
|
|
||||||
*
|
|
||||||
* @return array{bytes: int, approximate: bool}
|
|
||||||
*/
|
|
||||||
private function estimateBytes(Run $run, array $files): array {
|
|
||||||
if (($files['parent'] ?? null) !== null) {
|
|
||||||
try {
|
|
||||||
return ['bytes' => $this->snapshots->findOne($run->getTargetId(), (string)$files['parent'])->getBytes() ?? 0, 'approximate' => false];
|
|
||||||
} catch (\Throwable) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->select('numeric_id')->from('storages')
|
|
||||||
->where($qb->expr()->like('id', $qb->createNamedParameter('home::%')))
|
|
||||||
->orWhere($qb->expr()->like('id', $qb->createNamedParameter('object::user:%')));
|
|
||||||
$result = $qb->executeQuery();
|
|
||||||
$storages = [];
|
|
||||||
while (($id = $result->fetchOne()) !== false) {
|
|
||||||
$storages[] = (int)$id;
|
|
||||||
}
|
|
||||||
$result->closeCursor();
|
|
||||||
$bytes = 0;
|
|
||||||
foreach (array_chunk($storages, 500) as $chunk) {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->select($qb->func()->sum('size'))->from('filecache')
|
|
||||||
->where($qb->expr()->in('storage', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
|
|
||||||
->andWhere($qb->expr()->in('path_hash', $qb->createNamedParameter(array_map('md5', ['files', 'files_versions', 'files_trashbin']), IQueryBuilder::PARAM_STR_ARRAY)))
|
|
||||||
->andWhere($qb->expr()->gt('size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
|
|
||||||
$bytes += (int)$qb->executeQuery()->fetchOne();
|
|
||||||
}
|
|
||||||
return ['bytes' => $bytes, 'approximate' => true];
|
|
||||||
} catch (\Throwable) {
|
|
||||||
return ['bytes' => 0, 'approximate' => true];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function createRun(Target $target, string $label): Run {
|
|
||||||
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
$running = $this->runs->findRunning($target->getId(), self::KIND_FULL);
|
||||||
if ($running !== []) {
|
if ($running !== []) {
|
||||||
return $running[0];
|
return $running[0];
|
||||||
@@ -151,109 +84,36 @@ final class BackupService {
|
|||||||
/**
|
/**
|
||||||
* Do one step of a run until $deadline. Safe to call concurrently: a second caller finds the
|
* Do one step of a run until $deadline. Safe to call concurrently: a second caller finds the
|
||||||
* run locked and returns immediately.
|
* run locked and returns immediately.
|
||||||
*
|
|
||||||
* @param Repository|null $repo the location's repository, opened by an earlier step of this run
|
|
||||||
* in this process (see runToCompletion()); set when this step opens it
|
|
||||||
*/
|
*/
|
||||||
public function step(Run $run, float $deadline, ?Repository &$repo = null): Run {
|
public function step(Run $run, float $deadline): Run {
|
||||||
// Leases expire on their own: a step killed by a time limit or OOM blocks nothing for long.
|
$lockKey = 'ng_backup/run/' . $run->getId();
|
||||||
// A step can run well past its deadline (the commit of a large snapshot writes its tree and
|
|
||||||
// index), so the lease holds at least 15 minutes; should it still expire and another process
|
|
||||||
// take the run over, the checks below keep the two from overwriting each other.
|
|
||||||
$ttl = (int)min(3600, max(900, $deadline - microtime(true) + 600));
|
|
||||||
// No new step once a cancel has begun: it waits for the step in progress, if any.
|
|
||||||
try {
|
try {
|
||||||
$this->leases->release($this->leases->acquire(self::cancelKey($run->getId()), LeaseService::SHARED, 60));
|
$this->locking->acquireLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
|
||||||
} catch (LockedException) {
|
} catch (LockedException) {
|
||||||
return $run;
|
return $run;
|
||||||
}
|
}
|
||||||
try {
|
|
||||||
$runLease = $this->leases->acquire('run/' . $run->getId(), LeaseService::EXCLUSIVE, $ttl);
|
|
||||||
} catch (LockedException) {
|
|
||||||
return $run; // another process is doing this step
|
|
||||||
}
|
|
||||||
// Shared lease on the location while this step writes: a cleanup (exclusive) cannot run now.
|
|
||||||
try {
|
|
||||||
$targetLease = $this->leases->acquire('ng_backup/target/' . $run->getTargetId(), LeaseService::SHARED, $ttl);
|
|
||||||
} catch (LockedException) {
|
|
||||||
$this->leases->release($runLease);
|
|
||||||
return $run;
|
|
||||||
}
|
|
||||||
$loadedAt = null;
|
|
||||||
$savedBefore = $run->getUpdatedAt();
|
|
||||||
try {
|
try {
|
||||||
$run = $this->runs->find($run->getId()); // fresh state after acquiring the lock
|
$run = $this->runs->find($run->getId()); // fresh state after acquiring the lock
|
||||||
if ($run->getStatus() !== Run::RUNNING || $run->getKind() !== self::KIND_FULL) {
|
if ($run->getStatus() !== Run::RUNNING || $run->getKind() !== self::KIND_FULL) {
|
||||||
return $run;
|
return $run;
|
||||||
}
|
}
|
||||||
$loadedAt = $run->getUpdatedAt();
|
|
||||||
$target = $this->targets->get((string)$run->getTargetId());
|
$target = $this->targets->get((string)$run->getTargetId());
|
||||||
// Reopen when another process took a step since ours (its packs are not in our copy of
|
$repo = $this->targets->repository($target);
|
||||||
// the index) or the location was edited (password, folder, size limit).
|
|
||||||
$settings = json_encode([$target->getBackend(), $target->getAuth(), $target->getOptions(),
|
|
||||||
$target->getBasePath(), $target->getRepositoryId(), $target->getMaxBytes()], JSON_THROW_ON_ERROR);
|
|
||||||
if ($repo === null || $loadedAt !== $savedBefore || ($this->openedWith[$repo] ?? null) !== $settings) {
|
|
||||||
// Opening reads the index listing, which takes minutes on a slow location with many
|
|
||||||
// packs: that time does not count, or each step would store one blob and stop.
|
|
||||||
$opening = microtime(true);
|
|
||||||
$repo = null;
|
|
||||||
$repo = $this->targets->repository($target);
|
|
||||||
$this->openedWith[$repo] = $settings;
|
|
||||||
$deadline += microtime(true) - $opening;
|
|
||||||
}
|
|
||||||
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
$state = json_decode($run->getState(), true, 512, JSON_THROW_ON_ERROR);
|
||||||
|
|
||||||
if ($run->getPhase() === 'db') {
|
if ($run->getPhase() === 'db') {
|
||||||
$dumper = new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_'));
|
@set_time_limit(0); // the dump is one transaction; it cannot be split across requests yet
|
||||||
$dbStats = $state['dbStats'] ?? [];
|
$dbStats = [];
|
||||||
// With system cron the whole dump usually fits in one step (one transaction, fully
|
$manifest = (new DbDumper($this->db->getInner(), $this->config->getSystemValueString('dbtableprefix', 'oc_')))->dump($repo, $dbStats);
|
||||||
// consistent). Under web cron it continues per step; the snapshot then says so.
|
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
|
||||||
$dump = $dumper->dumpStep($repo, $state['dump'] ?? $dumper->startDump(), $deadline, $dbStats);
|
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
|
||||||
$state['dbStats'] = $dbStats;
|
$state['dbStats'] = $dbStats + ['tables' => count($manifest['tables']),
|
||||||
if ($dump['done']) {
|
'rows' => array_sum(array_column($manifest['tables'], 'rows'))];
|
||||||
$manifest = $dumper->manifest($dump);
|
$run->setPhase('files');
|
||||||
$repo->putObject('db/' . $state['files']['snapshot'], json_encode($manifest, JSON_THROW_ON_ERROR));
|
|
||||||
$state['files']['meta']['db'] = 'db/' . $state['files']['snapshot'];
|
|
||||||
$state['files']['meta']['dbConsistent'] = $manifest['consistent'];
|
|
||||||
$state['dbStats'] += ['tables' => count($manifest['tables']), 'rows' => array_sum(array_column($manifest['tables'], 'rows')),
|
|
||||||
'steps' => $dump['steps'], 'consistent' => $manifest['consistent']];
|
|
||||||
unset($state['dump']);
|
|
||||||
$run->setPhase('files');
|
|
||||||
} else {
|
|
||||||
$state['dump'] = $dump;
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
$state['files']['estimate'] ??= $this->estimateBytes($run, $state['files']);
|
|
||||||
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
|
$state['files'] = BackupRun::step($repo, $state['files'], $deadline);
|
||||||
if ($state['files']['phase'] === 'commit') {
|
|
||||||
// Catalog write under the catalog lock, from a repository opened inside it: a per-user
|
|
||||||
// export or restore may have written the catalog since this step opened $repo.
|
|
||||||
try {
|
|
||||||
$this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300,
|
|
||||||
fn () => $this->targets->repository($target)->recordSnapshot((string)$state['files']['snapshot']));
|
|
||||||
$state['files']['phase'] = 'done';
|
|
||||||
} catch (LockedException|RepositoryException) {
|
|
||||||
// Another catalog write is in progress (or just raced us to the next
|
|
||||||
// generation): the next step retries this commit on the fresh catalog.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($state['files']['phase'] === 'done') {
|
if ($state['files']['phase'] === 'done') {
|
||||||
$this->recordSnapshot($run, $state['files']);
|
$this->recordSnapshot($run, $state['files']);
|
||||||
// The snapshot is committed: what follows only reports on it and must not turn
|
|
||||||
// the run into a failure.
|
|
||||||
try {
|
|
||||||
$this->alerts->storedBytes($target, $repo->storedBytes());
|
|
||||||
$stats = $state['files']['stats'];
|
|
||||||
if (($stats['unreadable'] ?? 0) > 0 || ($stats['changing'] ?? 0) > 0) {
|
|
||||||
$warnings = BackupRun::warnings($state['files']);
|
|
||||||
$this->logger->warning(sprintf('NG Backup run %d to %s: %d unreadable (missing from the snapshot): %s; %d changed while being read: %s',
|
|
||||||
$run->getId(), $target->getName(), $stats['unreadable'] ?? 0, implode(', ', $warnings['unreadable']),
|
|
||||||
$stats['changing'] ?? 0, implode(', ', $warnings['changing'])));
|
|
||||||
$this->alerts->runWarnings($run, $target->getName(), (int)($stats['unreadable'] ?? 0), (int)($stats['changing'] ?? 0), $warnings);
|
|
||||||
}
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$this->logger->warning('NG Backup run ' . $run->getId() . ': reporting on the finished backup failed: ' . $e->getMessage(), ['exception' => $e]);
|
|
||||||
}
|
|
||||||
$run->setStatus(Run::DONE);
|
$run->setStatus(Run::DONE);
|
||||||
$run->setPhase('done');
|
$run->setPhase('done');
|
||||||
$run->setFinishedAt(time());
|
$run->setFinishedAt(time());
|
||||||
@@ -261,57 +121,24 @@ final class BackupService {
|
|||||||
}
|
}
|
||||||
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
|
$run->setState(json_encode($state, JSON_THROW_ON_ERROR));
|
||||||
$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
|
$run->setStats(json_encode(['db' => $state['dbStats'] ?? null, 'files' => $state['files']['stats']], JSON_THROW_ON_ERROR));
|
||||||
// Another process may have taken the run over after this step's lease expired: then its
|
$run->setUpdatedAt(time());
|
||||||
// writes stand and this step's are dropped.
|
return $this->runs->update($run);
|
||||||
$saved = $this->runs->updateIfUnchanged($run, $loadedAt);
|
|
||||||
if ($saved === null) {
|
|
||||||
$repo = null; // the run is another process's now: its packs are not in our index
|
|
||||||
return $this->runs->find($run->getId());
|
|
||||||
}
|
|
||||||
return $saved;
|
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
$repo = null; // may hold blobs of a half-written pack: the next step opens it afresh
|
$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
||||||
if ($loadedAt === null) {
|
|
||||||
// The run could not even be loaded: there is nothing fresh to mark as failed, and
|
|
||||||
// the caller's copy may be stale.
|
|
||||||
$this->logger->error('NG Backup run ' . $run->getId() . ': step could not load the run: ' . $e->getMessage(), ['exception' => $e]);
|
|
||||||
return $run;
|
|
||||||
}
|
|
||||||
$run->setStatus(Run::FAILED);
|
$run->setStatus(Run::FAILED);
|
||||||
$run->setError($e->getMessage());
|
$run->setError($e->getMessage());
|
||||||
$run->setFinishedAt(time());
|
$run->setFinishedAt(time());
|
||||||
$failed = $this->runs->updateIfUnchanged($run, $loadedAt);
|
$run->setUpdatedAt(time());
|
||||||
if ($failed === null) {
|
return $this->runs->update($run);
|
||||||
// Another process that took the run over may have finished it: its result stands.
|
|
||||||
$this->logger->warning('NG Backup run ' . $run->getId() . ': a step that another process had taken over ended with: ' . $e->getMessage());
|
|
||||||
return $this->runs->find($run->getId());
|
|
||||||
}
|
|
||||||
$run = $failed;
|
|
||||||
$this->logger->error('NG Backup run ' . $run->getId() . ' failed: ' . $e->getMessage(), ['exception' => $e]);
|
|
||||||
try {
|
|
||||||
$this->alerts->runFailed($run, isset($target) ? $target->getName() : '#' . $run->getTargetId());
|
|
||||||
} catch (\Throwable) {
|
|
||||||
}
|
|
||||||
return $run;
|
|
||||||
} finally {
|
} finally {
|
||||||
$this->leases->release($targetLease);
|
$this->locking->releaseLock($lockKey, ILockingProvider::LOCK_EXCLUSIVE);
|
||||||
$this->leases->release($runLease);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Run to completion in this process (occ, system cron), reporting progress. */
|
/** Run to completion in this process (occ, system cron), reporting progress. */
|
||||||
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
|
public function runToCompletion(Run $run, ?callable $progress = null, float $stepSeconds = 30): Run {
|
||||||
$repo = null; // opened once, by the first step, and kept for the next ones
|
|
||||||
while ($run->getStatus() === Run::RUNNING) {
|
while ($run->getStatus() === Run::RUNNING) {
|
||||||
$before = $run->getUpdatedAt();
|
$run = $this->step($run, microtime(true) + $stepSeconds);
|
||||||
$run = $this->step($run, microtime(true) + $stepSeconds, $repo);
|
|
||||||
if ($run->getStatus() === Run::RUNNING && $run->getUpdatedAt() === $before) {
|
|
||||||
// Step was not possible (another process holds the run, or a crashed step's lease has
|
|
||||||
// not expired yet): wait instead of spinning.
|
|
||||||
sleep(5);
|
|
||||||
$run = $this->runs->find($run->getId());
|
|
||||||
$repo = null; // another process may have stored packs in the meantime
|
|
||||||
}
|
|
||||||
if ($progress !== null) {
|
if ($progress !== null) {
|
||||||
$progress($run);
|
$progress($run);
|
||||||
}
|
}
|
||||||
@@ -319,78 +146,14 @@ final class BackupService {
|
|||||||
return $run;
|
return $run;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
public function cancel(Run $run): Run {
|
||||||
* Stop a running backup. First no new step may start, then this waits (up to $wait seconds)
|
$run->setStatus(Run::CANCELLED);
|
||||||
* for the step in progress, so a step that is recording the snapshot finishes first. A process
|
$run->setFinishedAt(time());
|
||||||
* running the backup then finds it cancelled and stops. The packs it stored stay until the
|
$run->setUpdatedAt(time());
|
||||||
* next cleanup.
|
return $this->runs->update($run);
|
||||||
*
|
|
||||||
* A process of an earlier version does not hold back its next step and may never leave a gap:
|
|
||||||
* after $wait the run is marked cancelled while it works, unless it is recording the snapshot.
|
|
||||||
* Its step then fails to save and the process stops.
|
|
||||||
*/
|
|
||||||
public function cancel(Run $run, int $wait = 120): Run {
|
|
||||||
$id = $run->getId();
|
|
||||||
$until = time() + $wait;
|
|
||||||
$hold = null;
|
|
||||||
for ($try = 0; $hold === null; $try++) {
|
|
||||||
try {
|
|
||||||
$hold = $this->leases->acquire(self::cancelKey($id), LeaseService::EXCLUSIVE, $wait + 300);
|
|
||||||
} catch (LockedException) {
|
|
||||||
if ($try >= 20) {
|
|
||||||
throw new \RuntimeException("Run $id is being cancelled already");
|
|
||||||
}
|
|
||||||
usleep(100000); // a step checking whether it may start
|
|
||||||
}
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$lease = null;
|
|
||||||
while ($lease === null && time() < $until) {
|
|
||||||
try {
|
|
||||||
$lease = $this->leases->acquire('run/' . $id, LeaseService::EXCLUSIVE, 60);
|
|
||||||
} catch (LockedException) {
|
|
||||||
sleep(2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
for ($try = 0; $try < 10; $try++) {
|
|
||||||
$run = $this->runs->find($id);
|
|
||||||
if ($run->getStatus() !== Run::RUNNING) {
|
|
||||||
return $run;
|
|
||||||
}
|
|
||||||
$state = json_decode($run->getState(), true) ?: [];
|
|
||||||
if ($lease === null && ($state['files']['phase'] ?? '') === 'commit') {
|
|
||||||
throw new \RuntimeException("Run $id is recording its snapshot; try again in a minute");
|
|
||||||
}
|
|
||||||
$loadedAt = $run->getUpdatedAt();
|
|
||||||
$run->setStatus(Run::CANCELLED);
|
|
||||||
$run->setFinishedAt(time());
|
|
||||||
if (($cancelled = $this->runs->updateIfUnchanged($run, $loadedAt)) !== null) {
|
|
||||||
return $cancelled;
|
|
||||||
}
|
|
||||||
usleep(200000); // a step of another process saved just now
|
|
||||||
}
|
|
||||||
throw new \RuntimeException("Run $id keeps changing; try again");
|
|
||||||
} finally {
|
|
||||||
if ($lease !== null) {
|
|
||||||
$this->leases->release($lease);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
$this->leases->release($hold);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function cancelKey(int $runId): string {
|
|
||||||
return 'run/' . $runId . '/cancel';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function recordSnapshot(Run $run, array $files): void {
|
private function recordSnapshot(Run $run, array $files): void {
|
||||||
try {
|
|
||||||
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
|
|
||||||
return; // already recorded, by an earlier attempt at this step
|
|
||||||
} catch (DoesNotExistException) {
|
|
||||||
}
|
|
||||||
$s = new Snapshot();
|
$s = new Snapshot();
|
||||||
$s->setTargetId($run->getTargetId());
|
$s->setTargetId($run->getTargetId());
|
||||||
$s->setSnapshotId($files['snapshot']);
|
$s->setSnapshotId($files['snapshot']);
|
||||||
@@ -399,19 +162,7 @@ final class BackupService {
|
|||||||
$s->setCreatedAt(time());
|
$s->setCreatedAt(time());
|
||||||
$s->setFiles($files['stats']['files']);
|
$s->setFiles($files['stats']['files']);
|
||||||
$s->setBytes($files['stats']['bytes']);
|
$s->setBytes($files['stats']['bytes']);
|
||||||
try {
|
$this->snapshots->insert($s);
|
||||||
$this->snapshots->insert($s);
|
|
||||||
} catch (\OCP\DB\Exception $e) {
|
|
||||||
// Recorded by another process between the check above and this insert?
|
|
||||||
if ($e->getReason() !== \OCP\DB\Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$this->snapshots->findOne($run->getTargetId(), $files['snapshot']);
|
|
||||||
} catch (DoesNotExistException) {
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<string> */
|
/** @return list<string> */
|
||||||
@@ -426,15 +177,6 @@ final class BackupService {
|
|||||||
'data/*/cache',
|
'data/*/cache',
|
||||||
'data/*/uploads',
|
'data/*/uploads',
|
||||||
];
|
];
|
||||||
// SQLite: the database file is in the data directory; it is backed up by the logical dump,
|
|
||||||
// and copying the live file would give an inconsistent copy.
|
|
||||||
if ($this->config->getSystemValueString('dbtype') === 'sqlite3') {
|
|
||||||
$dbName = $this->config->getSystemValueString('dbname', 'owncloud');
|
|
||||||
$exclude[] = "data/$dbName.db";
|
|
||||||
$exclude[] = "data/$dbName.db-journal";
|
|
||||||
$exclude[] = "data/$dbName.db-wal";
|
|
||||||
$exclude[] = "data/$dbName.db-shm";
|
|
||||||
}
|
|
||||||
$logFile = $this->config->getSystemValueString('logfile', '');
|
$logFile = $this->config->getSystemValueString('logfile', '');
|
||||||
if ($logFile !== '' && str_starts_with($logFile, $dataDir . '/')) {
|
if ($logFile !== '' && str_starts_with($logFile, $dataDir . '/')) {
|
||||||
$exclude[] = 'data/' . substr($logFile, strlen($dataDir) + 1);
|
$exclude[] = 'data/' . substr($logFile, strlen($dataDir) + 1);
|
||||||
|
|||||||
+11
-241
@@ -22,15 +22,11 @@ use OCP\Security\ICrypto;
|
|||||||
* into the recovery kit, so backups can be restored on a fresh server with the passphrase.
|
* into the recovery kit, so backups can be restored on a fresh server with the passphrase.
|
||||||
* - The admin must confirm having stored the recovery kit before the first backup runs.
|
* - The admin must confirm having stored the recovery kit before the first backup runs.
|
||||||
*/
|
*/
|
||||||
final class KeyService {
|
class KeyService {
|
||||||
private const KEY_MASTER = 'master_key';
|
private const KEY_MASTER = 'master_key';
|
||||||
private const KEY_WRAPPED = 'master_key_wrapped';
|
private const KEY_WRAPPED = 'master_key_wrapped';
|
||||||
private const KEY_CONFIRMED = 'recovery_kit_confirmed';
|
private const KEY_CONFIRMED = 'recovery_kit_confirmed';
|
||||||
private const KEY_FINGERPRINT = 'master_key_fingerprint';
|
private const KEY_FINGERPRINT = 'master_key_fingerprint';
|
||||||
private const KEY_DELETE_DELAY = 'delete_delay_days';
|
|
||||||
private const KEY_KIT_VERSION = 'recovery_kit_version';
|
|
||||||
private const KEY_KIT_CODE = 'recovery_kit_code';
|
|
||||||
public const DEFAULT_DELETE_DELAY = 7;
|
|
||||||
|
|
||||||
private ?KeyRing $cached = null;
|
private ?KeyRing $cached = null;
|
||||||
|
|
||||||
@@ -44,15 +40,7 @@ final class KeyService {
|
|||||||
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true) !== '';
|
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true) !== '';
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
public function initialize(#[\SensitiveParameter] string $passphrase): void {
|
||||||
* @param int $deleteDelayDays how long forgotten snapshots stay in the trash on the location
|
|
||||||
* before their data is really deleted. Fixed here, at installation:
|
|
||||||
* there is deliberately no way to change it later from NG Backup.
|
|
||||||
*/
|
|
||||||
public function initialize(#[\SensitiveParameter] string $passphrase, int $deleteDelayDays = self::DEFAULT_DELETE_DELAY, string $label = 'Slot 1'): void {
|
|
||||||
if ($deleteDelayDays < 1 || $deleteDelayDays > 365) {
|
|
||||||
throw new \InvalidArgumentException('The deletion delay must be between 1 and 365 days');
|
|
||||||
}
|
|
||||||
if ($this->isInitialized()) {
|
if ($this->isInitialized()) {
|
||||||
throw new \RuntimeException('A backup key already exists');
|
throw new \RuntimeException('A backup key already exists');
|
||||||
}
|
}
|
||||||
@@ -62,95 +50,13 @@ final class KeyService {
|
|||||||
$keys = KeyRing::generate();
|
$keys = KeyRing::generate();
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
||||||
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
||||||
$this->cached = $keys;
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED,
|
||||||
$this->saveSlots([['slot' => 1, 'label' => self::cleanLabel($label), 'created' => time(), 'key' => $keys->wrap($passphrase)]]);
|
json_encode($keys->wrap($passphrase), JSON_THROW_ON_ERROR), true, true);
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
||||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, $deleteDelayDays, true);
|
|
||||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, 1, true);
|
|
||||||
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_CONFIRMED);
|
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_CONFIRMED);
|
||||||
$this->cached = $keys;
|
$this->cached = $keys;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Disaster recovery on a fresh installation: adopt the master key from a recovery kit
|
|
||||||
* (occ backup:key:kit on the original installation) instead of generating a new one.
|
|
||||||
* Afterwards occ backup:target:add opens the existing repository on the location with this
|
|
||||||
* key, instead of creating a new, empty one.
|
|
||||||
*
|
|
||||||
* @param array<string, mixed> $kit as produced by recoveryKit()
|
|
||||||
* @throws \OCA\NgBackup\Crypto\CryptoException wrong passphrase, or a damaged/foreign kit
|
|
||||||
*/
|
|
||||||
public function importRecoveryKit(array $kit, #[\SensitiveParameter] string $passphrase): void {
|
|
||||||
if ($this->isInitialized()) {
|
|
||||||
throw new \RuntimeException('A backup key already exists');
|
|
||||||
}
|
|
||||||
if (!isset($kit['wrapped_key'])) {
|
|
||||||
throw new \InvalidArgumentException('Not a NG Backup recovery kit file');
|
|
||||||
}
|
|
||||||
$keys = KeyRing::unwrapAny($kit['wrapped_key'], $passphrase);
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER,
|
|
||||||
$this->crypto->encrypt(base64_encode($keys->exportMasterKey())), true, true);
|
|
||||||
$this->cached = $keys;
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, json_encode($kit['wrapped_key'], JSON_THROW_ON_ERROR), true, true);
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_FINGERPRINT, $this->fingerprintOf($keys), true);
|
|
||||||
// Kits from before this field existed fall back to the default.
|
|
||||||
$delay = (int)($kit['delete_delay_days'] ?? self::DEFAULT_DELETE_DELAY);
|
|
||||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, ($delay >= 1 && $delay <= 365) ? $delay : self::DEFAULT_DELETE_DELAY, true);
|
|
||||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, (int)($kit['kit_version'] ?? 1), true);
|
|
||||||
// Having and successfully using a real kit file is stronger proof than the normal
|
|
||||||
// download-and-confirm flow it replaces.
|
|
||||||
$this->confirmRecoveryKit('cli-import');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The master key and the recovery kit code are stored encrypted with Nextcloud's own
|
|
||||||
* instance secret (ICrypto), not with themselves; the wrapped key is not (it is sealed with
|
|
||||||
* the admin's passphrase instead), but Nextcloud's own AppConfig additionally encrypts any
|
|
||||||
* "sensitive" value with the instance secret regardless, underneath whatever we store. So
|
|
||||||
* changing that secret (occ backup:restore:full merging config.php) would otherwise leave
|
|
||||||
* all three permanently undecryptable. Call $applySecretChange (which must be the only thing
|
|
||||||
* that changes the secret) through here instead: all three are decrypted first, then
|
|
||||||
* re-encrypted under whatever secret is active once $applySecretChange returns.
|
|
||||||
*
|
|
||||||
* Not crash-safe: a process kill between the delete and the re-save of one of these keys
|
|
||||||
* would leave it missing rather than merely stale. Disaster recovery is run once, by hand,
|
|
||||||
* right after occ maintenance:install; a dedicated two-phase write is not justified yet for
|
|
||||||
* that narrow a window.
|
|
||||||
*/
|
|
||||||
public function reencryptForSecretRotation(callable $applySecretChange): mixed {
|
|
||||||
$master = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true);
|
|
||||||
$wrapped = $this->appConfig->getValueString(Application::APP_ID, self::KEY_WRAPPED, '', true);
|
|
||||||
$kitCodeRaw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_KIT_CODE, '', true);
|
|
||||||
$kitCode = $kitCodeRaw !== '' ? json_decode($kitCodeRaw, true, 512, JSON_THROW_ON_ERROR) : null;
|
|
||||||
$masterPlain = $master !== '' ? $this->crypto->decrypt($master) : null;
|
|
||||||
$kitCodePlain = $kitCode !== null && isset($kitCode['code']) ? $this->crypto->decrypt($kitCode['code']) : null;
|
|
||||||
|
|
||||||
$result = $applySecretChange();
|
|
||||||
$this->cached = null; // the master key is about to be re-saved; do not keep the pre-rotation copy
|
|
||||||
|
|
||||||
// Nextcloud's own AppConfig additionally encrypts a "sensitive" value itself (tied to
|
|
||||||
// the same secret, underneath our own encrypt() above); setValueString() on an existing
|
|
||||||
// key reads the old value first to compare, which would try to decrypt it with the new
|
|
||||||
// secret and fail the same way. Delete first so there is nothing to compare against.
|
|
||||||
if ($masterPlain !== null) {
|
|
||||||
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_MASTER);
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_MASTER, $this->crypto->encrypt($masterPlain), true, true);
|
|
||||||
}
|
|
||||||
if ($wrapped !== '') {
|
|
||||||
// Already plaintext JSON here: getValueString() above transparently undid
|
|
||||||
// AppConfig's own sensitive-value encryption; there is no encrypt() of our own to
|
|
||||||
// redo (wrapping is sealed with the passphrase, not the instance secret).
|
|
||||||
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_WRAPPED);
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, $wrapped, true, true);
|
|
||||||
}
|
|
||||||
if ($kitCodePlain !== null) {
|
|
||||||
$kitCode['code'] = $this->crypto->encrypt($kitCodePlain);
|
|
||||||
$this->appConfig->deleteKey(Application::APP_ID, self::KEY_KIT_CODE);
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_KIT_CODE, json_encode($kitCode, JSON_THROW_ON_ERROR), true, true);
|
|
||||||
}
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function keyRing(): KeyRing {
|
public function keyRing(): KeyRing {
|
||||||
if ($this->cached === null) {
|
if ($this->cached === null) {
|
||||||
$enc = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true);
|
$enc = $this->appConfig->getValueString(Application::APP_ID, self::KEY_MASTER, '', true);
|
||||||
@@ -162,99 +68,9 @@ final class KeyService {
|
|||||||
return $this->cached;
|
return $this->cached;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Deletion delay in days, fixed at installation (installations from before this setting get the default once). */
|
|
||||||
public function deleteDelayDays(): int {
|
|
||||||
$days = $this->appConfig->getValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, 0, true);
|
|
||||||
if ($days === 0 && $this->isInitialized()) {
|
|
||||||
$days = self::DEFAULT_DELETE_DELAY;
|
|
||||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_DELETE_DELAY, $days, true);
|
|
||||||
}
|
|
||||||
return $days;
|
|
||||||
}
|
|
||||||
|
|
||||||
public const MAX_SLOTS = 3;
|
|
||||||
|
|
||||||
/** @return list<array{slot:int, label:string, created:int}> passphrase slots, without key material */
|
|
||||||
public function slots(): array {
|
|
||||||
return array_map(fn ($s) => ['slot' => $s['slot'], 'label' => $s['label'], 'created' => $s['created']], $this->wrappedKey()['slots']);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Add a passphrase slot (e.g. one per person who must be able to restore). Returns the slot number. */
|
|
||||||
public function addSlot(#[\SensitiveParameter] string $passphrase, string $label): int {
|
|
||||||
self::checkPassphrase($passphrase);
|
|
||||||
$slots = $this->wrappedKey()['slots'];
|
|
||||||
if (count($slots) >= self::MAX_SLOTS) {
|
|
||||||
throw new \InvalidArgumentException('At most ' . self::MAX_SLOTS . ' passphrase slots');
|
|
||||||
}
|
|
||||||
$used = array_column($slots, 'slot');
|
|
||||||
$n = min(array_diff(range(1, self::MAX_SLOTS), $used));
|
|
||||||
$slots[] = ['slot' => $n, 'label' => self::cleanLabel($label), 'created' => time(), 'key' => $this->keyRing()->wrap($passphrase)];
|
|
||||||
usort($slots, fn ($a, $b) => $a['slot'] <=> $b['slot']);
|
|
||||||
$this->saveSlots($slots);
|
|
||||||
return $n;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** New passphrase for a slot (e.g. its holder left). The master key and all backups stay the same. */
|
|
||||||
public function replaceSlot(int $slot, #[\SensitiveParameter] string $passphrase, ?string $label = null): void {
|
|
||||||
self::checkPassphrase($passphrase);
|
|
||||||
$slots = $this->wrappedKey()['slots'];
|
|
||||||
$found = false;
|
|
||||||
foreach ($slots as &$s) {
|
|
||||||
if ($s['slot'] === $slot) {
|
|
||||||
$s['key'] = $this->keyRing()->wrap($passphrase);
|
|
||||||
$s['created'] = time();
|
|
||||||
if ($label !== null) {
|
|
||||||
$s['label'] = self::cleanLabel($label);
|
|
||||||
}
|
|
||||||
$found = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
unset($s);
|
|
||||||
if (!$found) {
|
|
||||||
throw new \InvalidArgumentException("There is no slot $slot");
|
|
||||||
}
|
|
||||||
$this->saveSlots($slots);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function removeSlot(int $slot): void {
|
|
||||||
$slots = array_values(array_filter($this->wrappedKey()['slots'], fn ($s) => $s['slot'] !== $slot));
|
|
||||||
if (count($slots) === count($this->wrappedKey()['slots'])) {
|
|
||||||
throw new \InvalidArgumentException("There is no slot $slot");
|
|
||||||
}
|
|
||||||
if ($slots === []) {
|
|
||||||
throw new \InvalidArgumentException('The last passphrase slot cannot be removed');
|
|
||||||
}
|
|
||||||
$this->saveSlots($slots);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Every change to the slots asks for a new recovery kit (and its confirmation). */
|
|
||||||
private function saveSlots(array $slots): void {
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_WRAPPED, json_encode(['slots' => $slots], JSON_THROW_ON_ERROR), true, true);
|
|
||||||
$this->appConfig->setValueInt(Application::APP_ID, self::KEY_KIT_VERSION, $this->kitVersion() + 1, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function checkPassphrase(string $passphrase): void {
|
|
||||||
if (strlen($passphrase) < 12) {
|
|
||||||
throw new \InvalidArgumentException('The passphrase must be at least 12 characters');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function cleanLabel(string $label): string {
|
|
||||||
$label = trim(preg_replace('/[\x00-\x1f]/', '', $label) ?? '');
|
|
||||||
return mb_substr($label === '' ? 'Slot' : $label, 0, 64);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function kitVersion(): int {
|
|
||||||
return $this->appConfig->getValueInt(Application::APP_ID, self::KEY_KIT_VERSION, 1, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Passphrase-wrapped copy of the master key (for repository configs and the recovery kit). */
|
/** Passphrase-wrapped copy of the master key (for repository configs and the recovery kit). */
|
||||||
public function wrappedKey(): array {
|
public function wrappedKey(): array {
|
||||||
$data = json_decode($this->appConfig->getValueString(Application::APP_ID, self::KEY_WRAPPED, '{}', true), true, 512, JSON_THROW_ON_ERROR);
|
return json_decode($this->appConfig->getValueString(Application::APP_ID, self::KEY_WRAPPED, '{}', true), true, 512, JSON_THROW_ON_ERROR);
|
||||||
if (!isset($data['slots'])) {
|
|
||||||
$data = ['slots' => $data === [] ? [] : [['slot' => 1, 'label' => 'Slot 1', 'created' => 0, 'key' => $data]]];
|
|
||||||
}
|
|
||||||
return $data;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Short identifier of the key, shown in the UI and the kit (not secret). */
|
/** Short identifier of the key, shown in the UI and the kit (not secret). */
|
||||||
@@ -262,75 +78,29 @@ final class KeyService {
|
|||||||
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_FINGERPRINT, '', true);
|
return $this->appConfig->getValueString(Application::APP_ID, self::KEY_FINGERPRINT, '', true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return array{version:int, app:string, fingerprint:string, kit_version:int, delete_delay_days:int, confirmation_code:string, created:string, wrapped_key:array, slots:list<array{slot:int, label:string, created:int}>, instructions:string} */
|
/** @return array{version:int, app:string, fingerprint:string, created:string, wrapped_key:array, instructions:string} */
|
||||||
public function recoveryKit(): array {
|
public function recoveryKit(): array {
|
||||||
return [
|
return [
|
||||||
'version' => 1,
|
'version' => 1,
|
||||||
'app' => 'NG Backup',
|
'app' => 'NG Backup',
|
||||||
'fingerprint' => $this->fingerprint(),
|
'fingerprint' => $this->fingerprint(),
|
||||||
'kit_version' => $this->kitVersion(),
|
|
||||||
'delete_delay_days' => $this->deleteDelayDays(),
|
|
||||||
'confirmation_code' => $this->kitCode(),
|
|
||||||
'created' => gmdate('c'),
|
'created' => gmdate('c'),
|
||||||
'wrapped_key' => $this->wrappedKey(),
|
'wrapped_key' => $this->wrappedKey(),
|
||||||
'slots' => $this->slots(),
|
'instructions' => 'Keep this file and your passphrase safe, outside this server. To restore on a new server: install NG Backup, '
|
||||||
'instructions' => 'Keep this file safe, outside this server. Any one of the passphrases of the slots listed here opens it. '
|
. 'add the backup location, and enter this kit with your passphrase. Without both, the backups cannot be decrypted.',
|
||||||
. 'To restore on a new server: install NG Backup, add the backup location and enter this kit with one passphrase. '
|
|
||||||
. 'Without the kit (or the location) and a passphrase, the backups cannot be decrypted.',
|
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Short code printed in the current kit file. Typing it back when confirming proves the
|
|
||||||
* admin actually has the downloaded kit. A new code per kit version; the server only keeps
|
|
||||||
* its own copy encrypted with the instance secret.
|
|
||||||
*/
|
|
||||||
private function kitCode(): string {
|
|
||||||
$stored = json_decode($this->appConfig->getValueString(Application::APP_ID, self::KEY_KIT_CODE, '{}', true), true) ?: [];
|
|
||||||
if (($stored['version'] ?? 0) === $this->kitVersion() && isset($stored['code'])) {
|
|
||||||
return $this->crypto->decrypt($stored['code']);
|
|
||||||
}
|
|
||||||
$alphabet = 'ABCDEFGHJKLMNPQRSTUVWXYZ23456789'; // no 0/O, 1/I
|
|
||||||
$code = '';
|
|
||||||
for ($i = 0; $i < 8; $i++) {
|
|
||||||
$code .= $alphabet[random_int(0, strlen($alphabet) - 1)];
|
|
||||||
}
|
|
||||||
$code = substr($code, 0, 4) . '-' . substr($code, 4);
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_KIT_CODE,
|
|
||||||
json_encode(['version' => $this->kitVersion(), 'code' => $this->crypto->encrypt($code)], JSON_THROW_ON_ERROR), true, true);
|
|
||||||
return $code;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function checkKitCode(string $code): bool {
|
|
||||||
$stored = json_decode($this->appConfig->getValueString(Application::APP_ID, self::KEY_KIT_CODE, '{}', true), true) ?: [];
|
|
||||||
if (($stored['version'] ?? 0) !== $this->kitVersion() || !isset($stored['code'])) {
|
|
||||||
return false; // no kit downloaded for this version yet
|
|
||||||
}
|
|
||||||
$normalise = static fn (string $c) => strtoupper(preg_replace('/[^A-Za-z0-9]/', '', $c) ?? '');
|
|
||||||
return hash_equals($normalise($this->crypto->decrypt($stored['code'])), $normalise($code));
|
|
||||||
}
|
|
||||||
|
|
||||||
public function confirmRecoveryKit(string $uid): void {
|
public function confirmRecoveryKit(string $uid): void {
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_CONFIRMED,
|
$this->appConfig->setValueString(Application::APP_ID, self::KEY_CONFIRMED,
|
||||||
json_encode(['uid' => $uid, 'time' => time(), 'fingerprint' => $this->fingerprint(), 'kit_version' => $this->kitVersion()], JSON_THROW_ON_ERROR), true);
|
json_encode(['uid' => $uid, 'time' => time(), 'fingerprint' => $this->fingerprint()], JSON_THROW_ON_ERROR), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** Confirmation for the current key, or null. */
|
||||||
* Whether a recovery kit for this key was ever confirmed (any passphrase version). Backups
|
|
||||||
* keep running after a passphrase change, so changing it cannot be used to stop backups; the
|
|
||||||
* missing re-confirmation is reported as an error in the admin overview instead.
|
|
||||||
*/
|
|
||||||
public function everConfirmed(): bool {
|
|
||||||
$raw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_CONFIRMED, '', true);
|
|
||||||
$data = $raw === '' ? null : json_decode($raw, true);
|
|
||||||
return $data !== null && ($data['fingerprint'] ?? '') === $this->fingerprint();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Confirmation for the current key and kit version, or null. */
|
|
||||||
public function recoveryKitConfirmation(): ?array {
|
public function recoveryKitConfirmation(): ?array {
|
||||||
$raw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_CONFIRMED, '', true);
|
$raw = $this->appConfig->getValueString(Application::APP_ID, self::KEY_CONFIRMED, '', true);
|
||||||
$data = $raw === '' ? null : json_decode($raw, true);
|
$data = $raw === '' ? null : json_decode($raw, true);
|
||||||
return ($data !== null && ($data['fingerprint'] ?? '') === $this->fingerprint() && ($data['kit_version'] ?? 1) === $this->kitVersion()) ? $data : null;
|
return ($data !== null && ($data['fingerprint'] ?? '') === $this->fingerprint()) ? $data : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function fingerprintOf(KeyRing $keys): string {
|
private function fingerprintOf(KeyRing $keys): string {
|
||||||
|
|||||||
@@ -1,61 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Passphrase slots (like LUKS): up to three passphrases, each opening the same master key, so
|
|
||||||
* that one person leaving or dying does not lock the company out of its backups. Every change
|
|
||||||
* is written to all locations and announced to all administrators; a new recovery kit must be
|
|
||||||
* downloaded and confirmed afterwards (backups keep running meanwhile).
|
|
||||||
*/
|
|
||||||
final class KeySlotService {
|
|
||||||
public function __construct(
|
|
||||||
private KeyService $keys,
|
|
||||||
private TargetService $targets,
|
|
||||||
private AlertService $alerts,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public function add(#[\SensitiveParameter] string $passphrase, string $label): int {
|
|
||||||
$slot = $this->keys->addSlot($passphrase, $label);
|
|
||||||
$this->apply('slot_added', $slot);
|
|
||||||
return $slot;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function replace(int $slot, #[\SensitiveParameter] string $passphrase, ?string $label = null): void {
|
|
||||||
$this->keys->replaceSlot($slot, $passphrase, $label);
|
|
||||||
$this->apply('slot_replaced', $slot);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function remove(int $slot): void {
|
|
||||||
$this->keys->removeSlot($slot);
|
|
||||||
$this->apply('slot_removed', $slot);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function apply(string $event, int $slot): void {
|
|
||||||
$wrapped = $this->keys->wrappedKey();
|
|
||||||
$failed = [];
|
|
||||||
foreach ($this->targets->list() as $t) {
|
|
||||||
try {
|
|
||||||
$this->targets->repository($t)->updateWrappedKey($wrapped);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$failed[] = $t->getName();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$label = '';
|
|
||||||
foreach ($this->keys->slots() as $s) {
|
|
||||||
if ($s['slot'] === $slot) {
|
|
||||||
$label = $s['label'];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$this->alerts->securityEvent($event, ['slot' => (string)$slot, 'label' => $label]);
|
|
||||||
if ($failed !== []) {
|
|
||||||
throw new \RuntimeException('Saved, but these locations could not be updated (they still accept the previous passphrases): ' . implode(', ', $failed));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
use OCP\IDBConnection;
|
|
||||||
use OCP\Lock\ILockingProvider;
|
|
||||||
use OCP\Lock\LockedException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Shared/exclusive leases with an expiry, stored in ngb_leases.
|
|
||||||
*
|
|
||||||
* Nextcloud's locks live up to an hour; a PHP process killed while holding one (web cron time
|
|
||||||
* limit, OOM, kill -9) would block a location that long. A lease expires on its own unless its
|
|
||||||
* holder keeps refreshing it. Nextcloud's lock is only held for the few milliseconds in which a
|
|
||||||
* lease is checked and written.
|
|
||||||
*/
|
|
||||||
final class LeaseService {
|
|
||||||
public const SHARED = 's';
|
|
||||||
public const EXCLUSIVE = 'x';
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private IDBConnection $db,
|
|
||||||
private ILockingProvider $locking,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return string lease id; throws LockedException when it conflicts with a live lease */
|
|
||||||
public function acquire(string $name, string $mode, int $ttl): string {
|
|
||||||
$guard = 'ng_backup/lease/' . $name;
|
|
||||||
$this->locking->acquireLock($guard, ILockingProvider::LOCK_EXCLUSIVE);
|
|
||||||
try {
|
|
||||||
$now = time();
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->delete('ngb_leases')
|
|
||||||
->where($qb->expr()->eq('name', $qb->createNamedParameter($name)))
|
|
||||||
->andWhere($qb->expr()->lt('expires_at', $qb->createNamedParameter($now)));
|
|
||||||
$qb->executeStatement();
|
|
||||||
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->select('mode')->from('ngb_leases')->where($qb->expr()->eq('name', $qb->createNamedParameter($name)));
|
|
||||||
$modes = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
|
|
||||||
if ($modes !== [] && ($mode === self::EXCLUSIVE || in_array(self::EXCLUSIVE, $modes, true))) {
|
|
||||||
throw new LockedException($name);
|
|
||||||
}
|
|
||||||
$id = bin2hex(random_bytes(16));
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->insert('ngb_leases')->values([
|
|
||||||
'id' => $qb->createNamedParameter($id),
|
|
||||||
'name' => $qb->createNamedParameter($name),
|
|
||||||
'mode' => $qb->createNamedParameter($mode),
|
|
||||||
'expires_at' => $qb->createNamedParameter($now + $ttl),
|
|
||||||
]);
|
|
||||||
$qb->executeStatement();
|
|
||||||
return $id;
|
|
||||||
} finally {
|
|
||||||
$this->locking->releaseLock($guard, ILockingProvider::LOCK_EXCLUSIVE);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Extend a lease while its holder is still working. */
|
|
||||||
public function refresh(string $id, int $ttl): void {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->update('ngb_leases')->set('expires_at', $qb->createNamedParameter(time() + $ttl))
|
|
||||||
->where($qb->expr()->eq('id', $qb->createNamedParameter($id)));
|
|
||||||
$qb->executeStatement();
|
|
||||||
}
|
|
||||||
|
|
||||||
public function release(string $id): void {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->delete('ngb_leases')->where($qb->expr()->eq('id', $qb->createNamedParameter($id)));
|
|
||||||
$qb->executeStatement();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Wraps $refresh so calling it often (per file, per row batch) only touches the database every $seconds. */
|
|
||||||
public static function throttled(callable $refresh, int $seconds = 60): \Closure {
|
|
||||||
$last = time();
|
|
||||||
return static function () use ($refresh, $seconds, &$last): void {
|
|
||||||
if (time() - $last >= $seconds) {
|
|
||||||
$refresh();
|
|
||||||
$last = time();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Run $fn under a lease; $refresh() can be called during long work. */
|
|
||||||
public function with(string $name, string $mode, int $ttl, callable $fn): mixed {
|
|
||||||
$id = $this->acquire($name, $mode, $ttl);
|
|
||||||
try {
|
|
||||||
return $fn(fn () => $this->refresh($id, $ttl));
|
|
||||||
} finally {
|
|
||||||
$this->release($id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,240 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
use OCA\NgBackup\AppInfo\Application;
|
|
||||||
use OCA\NgBackup\Db\RunMapper;
|
|
||||||
use OCA\NgBackup\Db\SnapshotMapper;
|
|
||||||
use OCA\NgBackup\Db\Target;
|
|
||||||
use OCP\IAppConfig;
|
|
||||||
use OCP\Lock\ILockingProvider;
|
|
||||||
use OCP\Lock\LockedException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Applies the retention policy to a location: forgets snapshots the policy does not keep and
|
|
||||||
* prunes data no remaining snapshot uses. Never runs on append-only locations, never while a
|
|
||||||
* backup to the same location is running (the location lock is shared with BackupService).
|
|
||||||
*/
|
|
||||||
final class PruneService {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private RunMapper $runs,
|
|
||||||
private SnapshotMapper $snapshots,
|
|
||||||
private IAppConfig $appConfig,
|
|
||||||
private ILockingProvider $locking,
|
|
||||||
private AlertService $alerts,
|
|
||||||
private KeyService $keys,
|
|
||||||
private LeaseService $leases,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public function untrash(Target $target, string $snapshotId): void {
|
|
||||||
$lease = $this->lockExclusive($target, 300);
|
|
||||||
try {
|
|
||||||
$this->doUntrash($target, $snapshotId);
|
|
||||||
} finally {
|
|
||||||
$this->leases->release($lease);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run $fn under the location's exclusive lock (to create a run atomically with respect to a
|
|
||||||
* cleanup, which takes the same lock and then refuses while a run exists).
|
|
||||||
*/
|
|
||||||
public function underStartLock(Target $target, callable $fn): mixed {
|
|
||||||
try {
|
|
||||||
return $this->leases->with(self::lockKey($target), LeaseService::EXCLUSIVE, 60, fn () => $fn());
|
|
||||||
} catch (LockedException) {
|
|
||||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Exclusive lock on the location, then make sure no run of any kind (backup, restore, export)
|
|
||||||
* is in progress: their data may not be referenced by a snapshot yet, or is being read.
|
|
||||||
*/
|
|
||||||
private function lockExclusive(Target $target, int $ttl): string {
|
|
||||||
try {
|
|
||||||
$lease = $this->leases->acquire(self::lockKey($target), LeaseService::EXCLUSIVE, $ttl);
|
|
||||||
} catch (LockedException) {
|
|
||||||
throw new \RuntimeException('A backup, restore or cleanup of ' . $target->getName() . ' is in progress; try again later');
|
|
||||||
}
|
|
||||||
if ($this->runs->findRunning($target->getId()) !== []) {
|
|
||||||
$this->leases->release($lease);
|
|
||||||
throw new \RuntimeException('A backup or restore of ' . $target->getName() . ' is running; try again later');
|
|
||||||
}
|
|
||||||
return $lease;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function doUntrash(Target $target, string $snapshotId): void {
|
|
||||||
$repo = $this->targets->repository($target);
|
|
||||||
if (!array_key_exists($snapshotId, $repo->trash())) {
|
|
||||||
throw new \InvalidArgumentException('That snapshot is not in the trash (or was already deleted)');
|
|
||||||
}
|
|
||||||
$repo->untrash($snapshotId);
|
|
||||||
// Put it back in the local snapshot list as well.
|
|
||||||
$meta = $repo->snapshotMeta($snapshotId);
|
|
||||||
$s = new \OCA\NgBackup\Db\Snapshot();
|
|
||||||
$s->setTargetId($target->getId());
|
|
||||||
$s->setSnapshotId($snapshotId);
|
|
||||||
$s->setKind((string)($meta['kind'] ?? 'full'));
|
|
||||||
$s->setLabel(($meta['label'] ?? '') !== '' ? $meta['label'] : null);
|
|
||||||
$s->setCreatedAt((int)strtotime((string)($meta['time'] ?? 'now')));
|
|
||||||
$s->setFiles((int)($meta['stats']['files'] ?? 0));
|
|
||||||
$s->setBytes((int)($meta['stats']['bytes'] ?? 0));
|
|
||||||
try {
|
|
||||||
$this->snapshots->insert($s);
|
|
||||||
} catch (\Throwable) {
|
|
||||||
}
|
|
||||||
$this->alerts->securityEvent('snapshot_untrashed', ['target' => $target->getName(), 'snapshot' => $snapshotId]);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function lockKey(Target $target): string {
|
|
||||||
return 'ng_backup/target/' . $target->getId();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A separate, short-lived exclusive lock serializing writes to the catalog itself. The
|
|
||||||
* per-target lock above only keeps readers/writers out while a cleanup runs; it does not
|
|
||||||
* stop two SHARED holders (e.g. two concurrent per-user backups, or one alongside a full
|
|
||||||
* backup) from both reading the same catalog generation and racing to write the next one,
|
|
||||||
* silently dropping whichever's change loses. The full-backup path avoids this today only
|
|
||||||
* because a Run row makes concurrent runs for the same target mutually exclusive; per-user
|
|
||||||
* backup/restore has no such row, so it takes this lock around the catalog write itself.
|
|
||||||
*/
|
|
||||||
public static function catalogLockKey(Target $target): string {
|
|
||||||
return self::lockKey($target) . '/catalog';
|
|
||||||
}
|
|
||||||
|
|
||||||
/** How many of each user's own exports (occ backup:user:backup) to keep, newest first. */
|
|
||||||
public function userExportRetention(): int {
|
|
||||||
return $this->appConfig->getValueInt(Application::APP_ID, 'user_export_retention', 3);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function setUserExportRetention(int $keep): void {
|
|
||||||
if ($keep < 1) {
|
|
||||||
throw new \InvalidArgumentException('Keep at least the last export');
|
|
||||||
}
|
|
||||||
$old = $this->userExportRetention();
|
|
||||||
$this->appConfig->setValueInt(Application::APP_ID, 'user_export_retention', $keep);
|
|
||||||
if ($old !== $keep) {
|
|
||||||
$this->alerts->securityEvent('user_export_retention_changed', ['keep' => (string)$keep]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Forgets a user's older exports once more than userExportRetention() exist for them,
|
|
||||||
* newest first; no trash, unlike a full snapshot (occ backup:user:backup is on-demand, not
|
|
||||||
* yet the primary safety net the deletion delay protects).
|
|
||||||
*
|
|
||||||
* @return array{kept: array<string, list<string>>, forgotten: list<string>}
|
|
||||||
*/
|
|
||||||
public function applyUserExports(Target $target, bool $dryRun = false): array {
|
|
||||||
if ($target->getAppendOnly()) {
|
|
||||||
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
|
|
||||||
}
|
|
||||||
$lease = $this->lockExclusive($target, 1800);
|
|
||||||
try {
|
|
||||||
$repo = $this->targets->repository($target);
|
|
||||||
$keep = $this->userExportRetention();
|
|
||||||
$byUser = [];
|
|
||||||
foreach ($repo->listUserExports() as $path) {
|
|
||||||
$uid = explode('/', $path)[1] ?? '';
|
|
||||||
$manifest = json_decode($repo->getObject($path), true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
$byUser[$uid][$path] = (int)strtotime((string)($manifest['time'] ?? '@0'));
|
|
||||||
}
|
|
||||||
$kept = [];
|
|
||||||
$forget = [];
|
|
||||||
foreach ($byUser as $uid => $times) {
|
|
||||||
arsort($times);
|
|
||||||
$i = 0;
|
|
||||||
foreach (array_keys($times) as $path) {
|
|
||||||
if ($i++ < $keep) {
|
|
||||||
$kept[$uid][] = $path;
|
|
||||||
} else {
|
|
||||||
$forget[] = $path;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!$dryRun) {
|
|
||||||
foreach ($forget as $path) {
|
|
||||||
$repo->forgetUserExport($path);
|
|
||||||
}
|
|
||||||
if ($forget !== []) {
|
|
||||||
$this->alerts->securityEvent('user_exports_forgotten', ['target' => $target->getName(), 'count' => (string)count($forget)], false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ['kept' => $kept, 'forgotten' => $forget];
|
|
||||||
} finally {
|
|
||||||
$this->leases->release($lease);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public function policy(): RetentionPolicy {
|
|
||||||
return RetentionPolicy::fromArray(json_decode($this->appConfig->getValueString(Application::APP_ID, 'retention', '{}'), true) ?: []);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function setPolicy(RetentionPolicy $policy): void {
|
|
||||||
if ($policy->last < 1) {
|
|
||||||
throw new \InvalidArgumentException('Keep at least the last snapshot');
|
|
||||||
}
|
|
||||||
$old = $this->policy()->toArray();
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, 'retention', json_encode($policy->toArray(), JSON_THROW_ON_ERROR));
|
|
||||||
if ($old !== $policy->toArray()) {
|
|
||||||
$this->alerts->securityEvent('retention_changed', ['policy' => json_encode($policy->toArray())]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Snapshots the policy drops go to the trash on the location; trash entries older than the
|
|
||||||
* deletion delay are purged, and only then is their data freed.
|
|
||||||
*
|
|
||||||
* @return array{kept: array<string, list<string>>, forgotten: list<string>, purged: list<string>, prune: array}
|
|
||||||
*/
|
|
||||||
public function apply(Target $target, bool $dryRun = false, bool $scheduled = false): array {
|
|
||||||
if ($target->getAppendOnly()) {
|
|
||||||
throw new \RuntimeException('Location ' . $target->getName() . ' is append-only: NG Backup does not delete anything there');
|
|
||||||
}
|
|
||||||
$lease = $this->lockExclusive($target, 1800);
|
|
||||||
try {
|
|
||||||
$repo = $this->targets->repository($target);
|
|
||||||
$times = [];
|
|
||||||
foreach ($repo->listSnapshots() as $id) {
|
|
||||||
$meta = $repo->snapshotMeta($id);
|
|
||||||
$times[$id] = (int)strtotime((string)($meta['time'] ?? '@0'));
|
|
||||||
}
|
|
||||||
$keep = $this->policy()->keep($times);
|
|
||||||
$forget = array_values(array_diff(array_keys($times), array_keys($keep)));
|
|
||||||
$actor = $scheduled ? 'retention policy' : $this->alerts->actor();
|
|
||||||
$purged = [];
|
|
||||||
if (!$dryRun) {
|
|
||||||
foreach ($forget as $id) {
|
|
||||||
$repo->forget($id, $actor);
|
|
||||||
try {
|
|
||||||
$this->snapshots->delete($this->snapshots->findOne($target->getId(), $id));
|
|
||||||
} catch (\Throwable) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Also removes leftovers of interrupted runs. A dry run counts the snapshots it would forget as gone.
|
|
||||||
$delay = $this->keys->deleteDelayDays();
|
|
||||||
if (!$dryRun && $forget !== []) {
|
|
||||||
// Scheduled cleanup follows the policy: audit log only, no notification every week.
|
|
||||||
$this->alerts->securityEvent('snapshots_forgotten', ['target' => $target->getName(), 'count' => (string)count($forget),
|
|
||||||
'until' => date('Y-m-d', time() + $delay * 86400)], !$scheduled);
|
|
||||||
}
|
|
||||||
if (!$dryRun) {
|
|
||||||
$purged = $repo->purgeTrash($delay * 86400);
|
|
||||||
}
|
|
||||||
$prune = $repo->prune($dryRun);
|
|
||||||
return ['kept' => $keep, 'forgotten' => $forget, 'purged' => $purged, 'prune' => $prune];
|
|
||||||
} finally {
|
|
||||||
$this->leases->release($lease);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,228 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Db\DbDumper;
|
|
||||||
use OCA\NgBackup\Db\Target;
|
|
||||||
use OCP\App\IAppManager;
|
|
||||||
use OCP\IConfig;
|
|
||||||
use OCP\IDBConnection;
|
|
||||||
use OCP\Lock\LockedException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Disaster recovery on a fresh Nextcloud installation (same major version): replace the
|
|
||||||
* database, restore the data directory and merge the backed-up config.php, see PLAN.md 3.7.
|
|
||||||
*
|
|
||||||
* Deliberately NOT handled yet (phase 3): restoring in place over a live, already-used
|
|
||||||
* installation. The admin is expected to run this once, right after occ maintenance:install and
|
|
||||||
* occ backup:key:import/backup:target:add, before anyone else touches the instance.
|
|
||||||
*/
|
|
||||||
final class RestoreFullService {
|
|
||||||
/**
|
|
||||||
* Config keys that must keep pointing at THIS installation's actual, already-working
|
|
||||||
* database and data directory (set by occ maintenance:install), never at the old server's.
|
|
||||||
*/
|
|
||||||
private const CONNECTION_KEYS = [
|
|
||||||
'dbtype', 'dbhost', 'dbport', 'dbname', 'dbuser', 'dbpassword', 'dbpersistent',
|
|
||||||
'dbtableprefix', 'dbdriveroptions', 'datadirectory',
|
|
||||||
];
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private RestoreService $restore,
|
|
||||||
private LeaseService $leases,
|
|
||||||
private IConfig $config,
|
|
||||||
private IDBConnection $db,
|
|
||||||
private IAppManager $appManager,
|
|
||||||
private KeyService $keys,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Whether this installation can receive $snapshotId: same Nextcloud major version, and every
|
|
||||||
* app that was enabled at backup time already installed here (its tables must exist before
|
|
||||||
* the database restore can write into them).
|
|
||||||
*
|
|
||||||
* @return array{ok:bool, currentMajor:string, snapshotMajor:string, missingApps:list<string>}
|
|
||||||
*/
|
|
||||||
public function checkCompatibility(Target $target, string $snapshotId): array {
|
|
||||||
$meta = $this->targets->repository($target)->snapshotMeta($snapshotId);
|
|
||||||
$currentMajor = explode('.', $this->config->getSystemValueString('version'))[0];
|
|
||||||
$snapshotMajor = explode('.', (string)($meta['nextcloud'] ?? ''))[0];
|
|
||||||
$enabled = array_flip($this->appManager->getEnabledApps());
|
|
||||||
$missingApps = array_values(array_diff(array_keys($meta['apps'] ?? []), array_keys($enabled)));
|
|
||||||
return [
|
|
||||||
'ok' => $currentMajor === $snapshotMajor && $missingApps === [],
|
|
||||||
'currentMajor' => $currentMajor,
|
|
||||||
'snapshotMajor' => $snapshotMajor,
|
|
||||||
'missingApps' => $missingApps,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return array{dbReport:array, dataFiles:int, configKeys:int, dbConsistent:bool} */
|
|
||||||
public function restoreFull(Target $target, string $snapshotId): array {
|
|
||||||
$check = $this->checkCompatibility($target, $snapshotId);
|
|
||||||
if (!$check['ok']) {
|
|
||||||
throw new \RuntimeException($this->describeBlockers($check));
|
|
||||||
}
|
|
||||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId) {
|
|
||||||
$beat = LeaseService::throttled($refresh);
|
|
||||||
$repo = $this->targets->repository($target);
|
|
||||||
$meta = $repo->snapshotMeta($snapshotId);
|
|
||||||
|
|
||||||
$dbReport = null;
|
|
||||||
$manifest = [];
|
|
||||||
if (isset($meta['db'])) {
|
|
||||||
$manifest = json_decode($repo->getObject($meta['db']), true, 512, JSON_THROW_ON_ERROR);
|
|
||||||
$prefix = $this->config->getSystemValueString('dbtableprefix', 'oc_');
|
|
||||||
// ng_backup's own tables (and its own rows in the shared appconfig table) must
|
|
||||||
// survive this restore untouched: they describe the backup/key/target that is
|
|
||||||
// actively performing it right now, on THIS installation, not the one that was
|
|
||||||
// backed up. Overwriting them mid-restore would pull the rug from under it.
|
|
||||||
foreach (array_keys($manifest['tables']) as $table) {
|
|
||||||
/** @var string $table */
|
|
||||||
if (str_starts_with($table, $prefix . 'ngb_')) {
|
|
||||||
unset($manifest['tables'][$table]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Sequences are keyed separately from tables; excluding a table does not exclude
|
|
||||||
// its sequence, which would otherwise rewind a retained ngb_* table's sequence
|
|
||||||
// below its current ids (PostgreSQL only -- see DbDumper::restoreSequences()).
|
|
||||||
foreach (array_keys($manifest['sequences'] ?? []) as $seq) {
|
|
||||||
/** @var string $seq */
|
|
||||||
if (str_starts_with((string)($manifest['sequences'][$seq]['table'] ?? ''), $prefix . 'ngb_')) {
|
|
||||||
unset($manifest['sequences'][$seq]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$ownAppConfig = $this->preserveOwnAppConfig();
|
|
||||||
$dumper = new DbDumper($this->db->getInner(), $prefix);
|
|
||||||
try {
|
|
||||||
$dumper->restore($repo, $manifest, $dbReport, $beat);
|
|
||||||
} finally {
|
|
||||||
$this->restoreOwnAppConfig($ownAppConfig);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read (but do not yet apply) the backed-up config.php before touching the data
|
|
||||||
// directory: once applied, 'secret' may differ from what ng_backup's own Target
|
|
||||||
// options were encrypted with (see restoreFull()'s docblock), and every further
|
|
||||||
// repository access through $this->targets would fail to decrypt them.
|
|
||||||
/** @var array<string, mixed> $backupConfig */
|
|
||||||
$backupConfig = $this->fetchBackupConfig($target, $snapshotId);
|
|
||||||
|
|
||||||
$dataDir = rtrim($this->config->getSystemValueString('datadirectory'), '/');
|
|
||||||
$dataFiles = $this->restore->restoreToDirectory($target, $snapshotId, 'data', $dataDir, stripPrefix: true, heartbeat: $beat);
|
|
||||||
|
|
||||||
// Last step: nothing after this may need $this->targets->repository($target) again.
|
|
||||||
if ($backupConfig !== []) {
|
|
||||||
$this->targets->reencryptOptionsAround(
|
|
||||||
fn (): mixed => $this->keys->reencryptForSecretRotation(fn (): mixed => $this->config->setSystemValues($backupConfig)),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return ['dbReport' => $dbReport ?? ['skipped' => [], 'restored' => []], 'dataFiles' => $dataFiles, 'configKeys' => count($backupConfig),
|
|
||||||
'dbConsistent' => (bool)($manifest['consistent'] ?? true)];
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return array<string, mixed> config.php's $CONFIG from the backup, minus the connection keys */
|
|
||||||
private function fetchBackupConfig(Target $target, string $snapshotId): array {
|
|
||||||
$tmp = sys_get_temp_dir() . '/ngb-restore-config-' . bin2hex(random_bytes(6));
|
|
||||||
try {
|
|
||||||
$this->restore->restoreToDirectory($target, $snapshotId, 'config', $tmp, stripPrefix: true);
|
|
||||||
$path = $tmp . '/config.php';
|
|
||||||
if (!is_file($path)) {
|
|
||||||
return []; // nothing to merge; the live config.php stays as occ maintenance:install made it
|
|
||||||
}
|
|
||||||
$backupConfig = (static function () use ($path): array {
|
|
||||||
$CONFIG = [];
|
|
||||||
include $path;
|
|
||||||
return $CONFIG;
|
|
||||||
})();
|
|
||||||
foreach (self::CONNECTION_KEYS as $key) {
|
|
||||||
unset($backupConfig[$key]);
|
|
||||||
}
|
|
||||||
return $backupConfig;
|
|
||||||
} finally {
|
|
||||||
$this->rmrf($tmp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @return list<array<string, mixed>> raw appconfig rows (appid, configkey, configvalue, type, lazy) */
|
|
||||||
private function preserveOwnAppConfig(): array {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->select('*')->from('appconfig')
|
|
||||||
->where($qb->expr()->eq('appid', $qb->createNamedParameter(\OCA\NgBackup\AppInfo\Application::APP_ID)));
|
|
||||||
/** @var list<array<string, mixed>> */
|
|
||||||
return $qb->executeQuery()->fetchAll();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param list<array<string, mixed>> $rows raw appconfig rows, as returned by preserveOwnAppConfig()
|
|
||||||
*
|
|
||||||
* Runs in its own transaction (DbDumper::restore() already committed its own by the time
|
|
||||||
* this is called): not a full guarantee against a crash spanning both, but means this step
|
|
||||||
* itself cannot leave ng_backup's appconfig rows half deleted, half reinserted.
|
|
||||||
*/
|
|
||||||
private function restoreOwnAppConfig(array $rows): void {
|
|
||||||
$this->db->beginTransaction();
|
|
||||||
try {
|
|
||||||
$qb = $this->db->getQueryBuilder();
|
|
||||||
$qb->delete('appconfig')
|
|
||||||
->where($qb->expr()->eq('appid', $qb->createNamedParameter(\OCA\NgBackup\AppInfo\Application::APP_ID)));
|
|
||||||
$qb->executeStatement();
|
|
||||||
foreach ($rows as $row) {
|
|
||||||
$ins = $this->db->getQueryBuilder();
|
|
||||||
$ins->insert('appconfig')->values([
|
|
||||||
'appid' => $ins->createNamedParameter($row['appid']),
|
|
||||||
'configkey' => $ins->createNamedParameter($row['configkey']),
|
|
||||||
'configvalue' => $ins->createNamedParameter($row['configvalue']),
|
|
||||||
'type' => $ins->createNamedParameter($row['type'], \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_INT),
|
|
||||||
'lazy' => $ins->createNamedParameter($row['lazy'], \OCP\DB\QueryBuilder\IQueryBuilder::PARAM_INT),
|
|
||||||
]);
|
|
||||||
$ins->executeStatement();
|
|
||||||
}
|
|
||||||
$this->db->commit();
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$this->db->rollBack();
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function rmrf(string $dir): void {
|
|
||||||
if (!is_dir($dir)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
foreach (new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($dir, \FilesystemIterator::SKIP_DOTS), \RecursiveIteratorIterator::CHILD_FIRST) as $f) {
|
|
||||||
$f->isDir() ? rmdir($f->getPathname()) : unlink($f->getPathname());
|
|
||||||
}
|
|
||||||
rmdir($dir);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function describeBlockers(array $check): string {
|
|
||||||
$parts = [];
|
|
||||||
if ($check['currentMajor'] !== $check['snapshotMajor']) {
|
|
||||||
$parts[] = "this installation is Nextcloud {$check['currentMajor']}, the backup is from {$check['snapshotMajor']} (same major version required)";
|
|
||||||
}
|
|
||||||
if ($check['missingApps'] !== []) {
|
|
||||||
$parts[] = 'install these apps first (their tables must exist before the database restore can use them): '
|
|
||||||
. implode(', ', array_map(fn (string $a) => "occ app:install $a", $check['missingApps']));
|
|
||||||
}
|
|
||||||
return implode('; ', $parts);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function withSharedLock(Target $target, callable $fn): mixed {
|
|
||||||
try {
|
|
||||||
// Refreshed during the database and data restore (per row batch, per blob), so a
|
|
||||||
// cleanup cannot take over the location while a long restore is still reading it.
|
|
||||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 7200, $fn);
|
|
||||||
} catch (LockedException) {
|
|
||||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -28,7 +28,7 @@ use OCP\Files\NotFoundException;
|
|||||||
* - merge: into the original place, existing files are overwritten (old content kept as a version)
|
* - merge: into the original place, existing files are overwritten (old content kept as a version)
|
||||||
* - replace: like merge, and files that are not in the snapshot are moved to the trash
|
* - replace: like merge, and files that are not in the snapshot are moved to the trash
|
||||||
*/
|
*/
|
||||||
final class RestoreService {
|
class RestoreService {
|
||||||
public const MODE_NEW_FOLDER = 'new-folder';
|
public const MODE_NEW_FOLDER = 'new-folder';
|
||||||
public const MODE_MERGE = 'merge';
|
public const MODE_MERGE = 'merge';
|
||||||
public const MODE_REPLACE = 'replace';
|
public const MODE_REPLACE = 'replace';
|
||||||
@@ -36,19 +36,9 @@ final class RestoreService {
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetService $targets,
|
private TargetService $targets,
|
||||||
private IRootFolder $root,
|
private IRootFolder $root,
|
||||||
private LeaseService $leases,
|
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Shared lock on the location while restoring: a cleanup (exclusive) must not delete what is being read. */
|
|
||||||
private function withSharedLock(Target $target, callable $fn, int $ttl = 600): mixed {
|
|
||||||
try {
|
|
||||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, $fn);
|
|
||||||
} catch (\OCP\Lock\LockedException) {
|
|
||||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is running; try again later');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Direct children (files and folders) of a path in a snapshot.
|
* Direct children (files and folders) of a path in a snapshot.
|
||||||
*
|
*
|
||||||
@@ -82,24 +72,7 @@ final class RestoreService {
|
|||||||
* @return array{restored:int, trashed:int, target:string}
|
* @return array{restored:int, trashed:int, target:string}
|
||||||
*/
|
*/
|
||||||
public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array {
|
public function restoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode = self::MODE_NEW_FOLDER, ?callable $progress = null): array {
|
||||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $logicalPath, $mode, $progress) {
|
if (!preg_match('#^data/([^/]+)/files(?:/(.*))?$#', trim($logicalPath, '/'), $m)) {
|
||||||
$last = time();
|
|
||||||
// Keep the lease alive while files are being written.
|
|
||||||
$beat = function (int $done, int $total, string $name) use ($refresh, $progress, &$last): void {
|
|
||||||
if (time() - $last >= 60) {
|
|
||||||
$refresh();
|
|
||||||
$last = time();
|
|
||||||
}
|
|
||||||
if ($progress !== null) {
|
|
||||||
$progress($done, $total, $name);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
return $this->doRestoreUserFiles($target, $snapshotId, $logicalPath, $mode, $beat);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private function doRestoreUserFiles(Target $target, string $snapshotId, string $logicalPath, string $mode, ?callable $progress): array {
|
|
||||||
if (!Repository::isSafePath($logicalPath) || !preg_match('#^data/([^/]+)/files(?:/(.*))?$#', $logicalPath, $m)) {
|
|
||||||
throw new \InvalidArgumentException('Expected a path like data/<user>/files/<folder or file>');
|
throw new \InvalidArgumentException('Expected a path like data/<user>/files/<folder or file>');
|
||||||
}
|
}
|
||||||
[$uid, $rel] = [$m[1], trim($m[2] ?? '', '/')];
|
[$uid, $rel] = [$m[1], trim($m[2] ?? '', '/')];
|
||||||
@@ -149,30 +122,12 @@ final class RestoreService {
|
|||||||
return ['restored' => $restored, 'trashed' => $trashed, 'target' => $uid . ':' . $userFolder->getRelativePath($dest->getPath())];
|
return ['restored' => $restored, 'trashed' => $trashed, 'target' => $uid . ':' . $userFolder->getRelativePath($dest->getPath())];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** Raw restore of any snapshot path to a local directory (administrators, disaster recovery). */
|
||||||
* Raw restore of any snapshot path to a local directory (administrators, disaster recovery).
|
public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory): int {
|
||||||
*
|
|
||||||
* @param bool $stripPrefix write into $directory itself instead of $directory/$prefix (see
|
|
||||||
* Repository::restore()); used to restore "data/" or "config/" straight into the
|
|
||||||
* real data/config directory for occ backup:restore:full.
|
|
||||||
*/
|
|
||||||
public function restoreToDirectory(Target $target, string $snapshotId, string $prefix, string $directory, bool $stripPrefix = false, ?callable $heartbeat = null): int {
|
|
||||||
if ($prefix !== '' && !Repository::isSafePath($prefix)) {
|
|
||||||
throw new \InvalidArgumentException('Invalid path: ' . $prefix);
|
|
||||||
}
|
|
||||||
if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) {
|
if (!is_dir($directory) && !mkdir($directory, 0750, true) && !is_dir($directory)) {
|
||||||
throw new \RuntimeException("Cannot create $directory");
|
throw new \RuntimeException("Cannot create $directory");
|
||||||
}
|
}
|
||||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $snapshotId, $prefix, $directory, $stripPrefix, $heartbeat): int {
|
return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/');
|
||||||
$own = LeaseService::throttled($refresh);
|
|
||||||
$beat = static function () use ($own, $heartbeat): void {
|
|
||||||
$own();
|
|
||||||
if ($heartbeat !== null) {
|
|
||||||
$heartbeat();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
return $this->targets->repository($target)->restore($snapshotId, $directory, trim($prefix, '/') === '' ? '' : trim($prefix, '/') . '/', $stripPrefix, $beat);
|
|
||||||
}, 3600);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void {
|
private function writeFile(Repository $repo, Folder $dest, string $relPath, array $entry): void {
|
||||||
|
|||||||
@@ -1,60 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Which snapshots to keep: the newest $last, plus the newest snapshot of each of the most recent
|
|
||||||
* $daily days, $weekly ISO weeks and $monthly months that have a snapshot (like restic/borg).
|
|
||||||
*/
|
|
||||||
final class RetentionPolicy {
|
|
||||||
public function __construct(
|
|
||||||
public readonly int $last = 3,
|
|
||||||
public readonly int $daily = 7,
|
|
||||||
public readonly int $weekly = 4,
|
|
||||||
public readonly int $monthly = 6,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public static function fromArray(array $a): self {
|
|
||||||
return new self((int)($a['last'] ?? 3), (int)($a['daily'] ?? 7), (int)($a['weekly'] ?? 4), (int)($a['monthly'] ?? 6));
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toArray(): array {
|
|
||||||
return ['last' => $this->last, 'daily' => $this->daily, 'weekly' => $this->weekly, 'monthly' => $this->monthly];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param array<string, int> $snapshots id => unix time
|
|
||||||
* @return array<string, list<string>> id => reasons it is kept (ids not in the result are removed)
|
|
||||||
*/
|
|
||||||
public function keep(array $snapshots): array {
|
|
||||||
arsort($snapshots);
|
|
||||||
$keep = [];
|
|
||||||
$i = 0;
|
|
||||||
foreach ($snapshots as $id => $time) {
|
|
||||||
if ($i++ < $this->last) {
|
|
||||||
$keep[$id][] = 'last';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
foreach (['daily' => 'Y-m-d', 'weekly' => 'o-W', 'monthly' => 'Y-m'] as $kind => $format) {
|
|
||||||
$seen = [];
|
|
||||||
foreach ($snapshots as $id => $time) {
|
|
||||||
$bucket = date($format, $time);
|
|
||||||
if (isset($seen[$bucket])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (count($seen) >= $this->$kind) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
$seen[$bucket] = true;
|
|
||||||
$keep[$id][] = $kind;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return $keep;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+20
-211
@@ -10,14 +10,10 @@ namespace OCA\NgBackup\Service;
|
|||||||
use OCA\NgBackup\Backend\ExternalStorageFactory;
|
use OCA\NgBackup\Backend\ExternalStorageFactory;
|
||||||
use OCA\NgBackup\Backend\IBackend;
|
use OCA\NgBackup\Backend\IBackend;
|
||||||
use OCA\NgBackup\Backend\StorageBackend;
|
use OCA\NgBackup\Backend\StorageBackend;
|
||||||
use OCA\NgBackup\Db\Snapshot;
|
|
||||||
use OCA\NgBackup\Db\SnapshotMapper;
|
|
||||||
use OCA\NgBackup\Db\Target;
|
use OCA\NgBackup\Db\Target;
|
||||||
use OCA\NgBackup\Db\TargetMapper;
|
use OCA\NgBackup\Db\TargetMapper;
|
||||||
use OCA\NgBackup\Repository\Repository;
|
use OCA\NgBackup\Repository\Repository;
|
||||||
use OCP\App\IAppManager;
|
use OCP\App\IAppManager;
|
||||||
use OCP\AppFramework\Db\DoesNotExistException;
|
|
||||||
use OCP\Files\Storage\IStorage;
|
|
||||||
use OCP\Security\ICrypto;
|
use OCP\Security\ICrypto;
|
||||||
use OCP\Server;
|
use OCP\Server;
|
||||||
|
|
||||||
@@ -26,19 +22,17 @@ use OCP\Server;
|
|||||||
* object is built only while it is used; nothing is mounted. files_external is loaded for its
|
* object is built only while it is used; nothing is mounted. files_external is loaded for its
|
||||||
* classes but its enabled/disabled state is left as the admin set it.
|
* classes but its enabled/disabled state is left as the admin set it.
|
||||||
*/
|
*/
|
||||||
final class TargetService {
|
class TargetService {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private TargetMapper $mapper,
|
private TargetMapper $mapper,
|
||||||
private KeyService $keys,
|
private KeyService $keys,
|
||||||
private ICrypto $crypto,
|
private ICrypto $crypto,
|
||||||
private IAppManager $appManager,
|
private IAppManager $appManager,
|
||||||
private \OCA\NgBackup\Db\DbIndexCache $indexCache,
|
private \OCA\NgBackup\Db\DbIndexCache $indexCache,
|
||||||
private \OCA\NgBackup\Db\AppConfigCatalogAnchor $anchor,
|
|
||||||
private SnapshotMapper $snapshots,
|
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<array{id:string, name:string, auth:list<array{id:string, name:string, parameters:array}>, parameters:array}> */
|
/** @return list<array{id:string, name:string, auth:list<string>, parameters:array}> */
|
||||||
public function availableBackends(): array {
|
public function availableBackends(): array {
|
||||||
return $this->factory()->describeBackends();
|
return $this->factory()->describeBackends();
|
||||||
}
|
}
|
||||||
@@ -51,15 +45,21 @@ final class TargetService {
|
|||||||
* @return array{target: Target, created: bool}
|
* @return array{target: Target, created: bool}
|
||||||
*/
|
*/
|
||||||
public function add(string $name, string $backend, string $auth, array $options, string $basePath): array {
|
public function add(string $name, string $backend, string $auth, array $options, string $basePath): array {
|
||||||
$this->validateName($name);
|
if (!preg_match('/^[A-Za-z0-9][A-Za-z0-9 _.-]{0,63}$/', $name)) {
|
||||||
$this->requireUsableBackend($backend);
|
throw new \InvalidArgumentException('Name: letters, digits, space, _ . - (max 64)');
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
$this->mapper->findByName($name);
|
||||||
|
throw new \InvalidArgumentException("A target named '$name' already exists");
|
||||||
|
} catch (\OCP\AppFramework\Db\DoesNotExistException) {
|
||||||
|
}
|
||||||
$keys = $this->keys->keyRing();
|
$keys = $this->keys->keyRing();
|
||||||
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath);
|
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath);
|
||||||
if ($storageBackend->exists('config')) {
|
if ($storageBackend->exists('config')) {
|
||||||
$repo = Repository::openWithKey($storageBackend, $keys, $this->indexCache, $this->anchor);
|
$repo = Repository::openWithKey($storageBackend, $keys);
|
||||||
$created = false;
|
$created = false;
|
||||||
} else {
|
} else {
|
||||||
$repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey(), $this->anchor);
|
$repo = Repository::initWithKey($storageBackend, $keys, $this->keys->wrappedKey());
|
||||||
$created = true;
|
$created = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,159 +77,7 @@ final class TargetService {
|
|||||||
$target->setBasePath(trim($basePath, '/'));
|
$target->setBasePath(trim($basePath, '/'));
|
||||||
$target->setRepositoryId($repo->id());
|
$target->setRepositoryId($repo->id());
|
||||||
$target->setCreatedAt(time());
|
$target->setCreatedAt(time());
|
||||||
$target = $this->mapper->insert($target);
|
return ['target' => $this->mapper->insert($target), 'created' => $created];
|
||||||
if (!$created) {
|
|
||||||
// Disaster recovery: occ backup:list reads this local cache, not the repository
|
|
||||||
// directly, so a snapshot made by another installation must be backfilled here too.
|
|
||||||
$this->adoptSnapshots($target, $repo);
|
|
||||||
}
|
|
||||||
return ['target' => $target, 'created' => $created];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Change how an existing location is reached (credentials, host, folder, even the connection
|
|
||||||
* type) or its name. The new settings must lead to the same repository: the history stays
|
|
||||||
* and a typo cannot silently start a new, empty repository. A null option value keeps the
|
|
||||||
* stored one, so the web interface never has to send stored passwords back.
|
|
||||||
*
|
|
||||||
* @param array<string, mixed> $options
|
|
||||||
*/
|
|
||||||
public function update(Target $target, string $name, string $backend, string $auth, array $options, string $basePath): Target {
|
|
||||||
$this->validateName($name, $target->getId());
|
|
||||||
$this->requireUsableBackend($backend);
|
|
||||||
$stored = $this->options($target);
|
|
||||||
foreach ($options as $key => $value) {
|
|
||||||
if ($value === null) {
|
|
||||||
if (array_key_exists($key, $stored)) {
|
|
||||||
$options[$key] = $stored[$key];
|
|
||||||
} else {
|
|
||||||
unset($options[$key]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Do not create the folder: a typo in the path must not leave an empty folder behind.
|
|
||||||
$storageBackend = $this->buildBackend($backend, $auth, $options, $basePath, false);
|
|
||||||
if (!$storageBackend->exists('config')) {
|
|
||||||
if (!$storageBackend->canConnect()) {
|
|
||||||
throw new \InvalidArgumentException('Cannot connect with these settings. Check the host, user name and password');
|
|
||||||
}
|
|
||||||
throw new \InvalidArgumentException("No NG Backup repository found in folder '" . trim($basePath, '/') . "'. To start an empty repository, add a new location instead");
|
|
||||||
}
|
|
||||||
$repo = Repository::openWithKey($storageBackend, $this->keys->keyRing(), $this->indexCache, $this->anchor);
|
|
||||||
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
|
||||||
throw new \InvalidArgumentException('These settings lead to a different repository than the one of this location');
|
|
||||||
}
|
|
||||||
$target->setName($name);
|
|
||||||
$target->setBackend($backend);
|
|
||||||
$target->setAuth($auth);
|
|
||||||
$target->setOptions($this->crypto->encrypt(json_encode($options, JSON_THROW_ON_ERROR)));
|
|
||||||
$target->setBasePath(trim($basePath, '/'));
|
|
||||||
return $this->mapper->update($target);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The stored options for an edit form: password-type values are replaced by null, which
|
|
||||||
* update() reads as "keep the stored value".
|
|
||||||
*
|
|
||||||
* @return array<string, mixed>
|
|
||||||
*/
|
|
||||||
public function editableOptions(Target $target): array {
|
|
||||||
$secret = [];
|
|
||||||
foreach ($this->availableBackends() as $b) {
|
|
||||||
if ($b['id'] !== $target->getBackend()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$params = $b['parameters'];
|
|
||||||
foreach ($b['auth'] as $a) {
|
|
||||||
if ($a['id'] === $target->getAuth()) {
|
|
||||||
$params += $a['parameters'];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
foreach ($params as $key => $p) {
|
|
||||||
if (($p['type'] ?? 0) === 2) {
|
|
||||||
$secret[$key] = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$options = $this->options($target);
|
|
||||||
foreach ($options as $key => $value) {
|
|
||||||
if (isset($secret[$key])) {
|
|
||||||
$options[$key] = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return $options;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function validateName(string $name, ?int $ownId = null): void {
|
|
||||||
if (!preg_match('/^[A-Za-z0-9][A-Za-z0-9 _.-]{0,63}$/', $name)) {
|
|
||||||
throw new \InvalidArgumentException('Name: letters, digits, space, _ . - (max 64)');
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$other = $this->mapper->findByName($name);
|
|
||||||
} catch (DoesNotExistException) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if ($other->getId() !== $ownId) {
|
|
||||||
throw new \InvalidArgumentException("A target named '$name' already exists");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function requireUsableBackend(string $backend): void {
|
|
||||||
// Without the native extension files_external falls back to the smbclient binary, which
|
|
||||||
// registers fine but cannot seek: the first backup then fails with "Seek failed". Refuse
|
|
||||||
// up front instead of letting the admin find out from a failed run.
|
|
||||||
if ($backend === 'smb' && !extension_loaded('smbclient')) {
|
|
||||||
throw new \InvalidArgumentException('The SMB backend needs the PHP smbclient extension (pecl install smbclient); the smbclient binary alone cannot seek and every backup would fail');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A target's options are encrypted with Nextcloud's own instance secret (ICrypto), not
|
|
||||||
* ng_backup's key, so changing that secret (occ backup:restore:full merging config.php)
|
|
||||||
* would otherwise leave every target permanently undecryptable. Call $applySecretChange
|
|
||||||
* (which must be the only thing that changes the secret) through here instead: every
|
|
||||||
* target's options are decrypted first, then re-encrypted under whatever secret is active
|
|
||||||
* once $applySecretChange returns.
|
|
||||||
*/
|
|
||||||
public function reencryptOptionsAround(callable $applySecretChange): mixed {
|
|
||||||
$plain = [];
|
|
||||||
foreach ($this->mapper->findAll() as $target) {
|
|
||||||
$plain[$target->getId()] = $this->options($target);
|
|
||||||
}
|
|
||||||
$result = $applySecretChange();
|
|
||||||
foreach ($this->mapper->findAll() as $target) {
|
|
||||||
if (!isset($plain[$target->getId()])) {
|
|
||||||
continue; // added by $applySecretChange itself; nothing to re-encrypt
|
|
||||||
}
|
|
||||||
$target->setOptions($this->crypto->encrypt(json_encode($plain[$target->getId()], JSON_THROW_ON_ERROR)));
|
|
||||||
$this->mapper->update($target);
|
|
||||||
}
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function adoptSnapshots(Target $target, Repository $repo): void {
|
|
||||||
foreach ($repo->listSnapshots() as $snapshotId) {
|
|
||||||
try {
|
|
||||||
$this->snapshots->findOne($target->getId(), $snapshotId);
|
|
||||||
continue; // already known locally
|
|
||||||
} catch (DoesNotExistException) {
|
|
||||||
}
|
|
||||||
// One unreadable/corrupt snapshot must not stop the rest of the location's
|
|
||||||
// snapshots from being backfilled, nor abort adding the target itself.
|
|
||||||
try {
|
|
||||||
$meta = $repo->snapshotMeta($snapshotId);
|
|
||||||
$s = new Snapshot();
|
|
||||||
$s->setTargetId($target->getId());
|
|
||||||
$s->setSnapshotId($snapshotId);
|
|
||||||
$s->setKind((string)($meta['kind'] ?? 'full'));
|
|
||||||
$s->setLabel(($meta['label'] ?? '') !== '' ? $meta['label'] : null);
|
|
||||||
$s->setCreatedAt((int)strtotime((string)($meta['time'] ?? 'now')));
|
|
||||||
$s->setFiles((int)($meta['stats']['files'] ?? 0));
|
|
||||||
$s->setBytes((int)($meta['stats']['bytes'] ?? 0));
|
|
||||||
$this->snapshots->insert($s);
|
|
||||||
} catch (\Throwable) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return list<Target> */
|
/** @return list<Target> */
|
||||||
@@ -238,22 +86,7 @@ final class TargetService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function get(string $nameOrId): Target {
|
public function get(string $nameOrId): Target {
|
||||||
try {
|
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
||||||
return ctype_digit($nameOrId) ? $this->mapper->find((int)$nameOrId) : $this->mapper->findByName($nameOrId);
|
|
||||||
} catch (DoesNotExistException) {
|
|
||||||
throw new \InvalidArgumentException("No such location: $nameOrId");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Size limit for the data stored on this location (null = none). */
|
|
||||||
public function setMaxBytes(Target $target, ?int $maxBytes): Target {
|
|
||||||
$target->setMaxBytes($maxBytes);
|
|
||||||
return $this->mapper->update($target);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function setAppendOnly(Target $target, bool $appendOnly): Target {
|
|
||||||
$target->setAppendOnly($appendOnly);
|
|
||||||
return $this->mapper->update($target);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Remove the target from NG Backup. The backups on the target itself are not touched. */
|
/** Remove the target from NG Backup. The backups on the target itself are not touched. */
|
||||||
@@ -266,11 +99,10 @@ final class TargetService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public function repository(Target $target): Repository {
|
public function repository(Target $target): Repository {
|
||||||
$repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache, $this->anchor);
|
$repo = Repository::openWithKey($this->backend($target), $this->keys->keyRing(), $this->indexCache);
|
||||||
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
if ($target->getRepositoryId() !== null && $repo->id() !== $target->getRepositoryId()) {
|
||||||
throw new \RuntimeException('The target now contains a different repository than when it was added');
|
throw new \RuntimeException('The target now contains a different repository than when it was added');
|
||||||
}
|
}
|
||||||
$repo->setMaxBytes($target->getMaxBytes());
|
|
||||||
return $repo;
|
return $repo;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -279,38 +111,15 @@ final class TargetService {
|
|||||||
return json_decode($this->crypto->decrypt($target->getOptions()), true, 512, JSON_THROW_ON_ERROR);
|
return json_decode($this->crypto->decrypt($target->getOptions()), true, 512, JSON_THROW_ON_ERROR);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function buildBackend(string $backend, string $auth, array $options, string $basePath, bool $createBase = true): StorageBackend {
|
private function buildBackend(string $backend, string $auth, array $options, string $basePath): StorageBackend {
|
||||||
$create = fn (): IStorage => $this->factory()->create($backend, $auth, $options);
|
return new StorageBackend($this->factory()->create($backend, $auth, $options), $basePath);
|
||||||
return new StorageBackend($create(), $basePath, $createBase, $create);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private ?ExternalStorageFactory $factory = null;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* files_external's connection code, without changing the app's state:
|
|
||||||
* - enabled: it is booted by Nextcloud; use its own BackendService;
|
|
||||||
* - disabled: do NOT boot it (booting registers its mount provider and listeners, which use
|
|
||||||
* its database tables; on installations where it was never enabled those do not exist).
|
|
||||||
* Only its classes are loaded, and a private BackendService gets its backends and auth
|
|
||||||
* mechanisms from its Application class.
|
|
||||||
*/
|
|
||||||
private function factory(): ExternalStorageFactory {
|
private function factory(): ExternalStorageFactory {
|
||||||
if ($this->factory !== null) {
|
// Load files_external's classes without enabling the app (its state stays as the admin set it).
|
||||||
return $this->factory;
|
if (!class_exists(\OCA\Files_External\Service\BackendService::class)) {
|
||||||
}
|
|
||||||
if ($this->appManager->isEnabledForAnyone('files_external')) {
|
|
||||||
$this->appManager->loadApp('files_external');
|
$this->appManager->loadApp('files_external');
|
||||||
return $this->factory = new ExternalStorageFactory(Server::get(\OCA\Files_External\Service\BackendService::class));
|
|
||||||
}
|
}
|
||||||
if (!class_exists(\OCA\Files_External\AppInfo\Application::class)) {
|
return new ExternalStorageFactory(Server::get(\OCA\Files_External\Service\BackendService::class));
|
||||||
$path = $this->appManager->getAppPath('files_external');
|
|
||||||
require_once $path . '/composer/autoload.php';
|
|
||||||
}
|
|
||||||
$service = new \OCA\Files_External\Service\BackendService(
|
|
||||||
Server::get(\OCP\IAppConfig::class), Server::get(\Psr\Log\LoggerInterface::class));
|
|
||||||
$app = new \OCA\Files_External\AppInfo\Application();
|
|
||||||
$service->registerBackendProvider($app);
|
|
||||||
$service->registerAuthMechanismProvider($app);
|
|
||||||
return $this->factory = new ExternalStorageFactory($service);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,66 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Db\Target;
|
|
||||||
use OCA\NgBackup\UserMigration\RepositoryExportDestination;
|
|
||||||
use OCA\NgBackup\UserMigration\UserMigrationServiceLocator;
|
|
||||||
use OCP\IUser;
|
|
||||||
use OCP\Lock\LockedException;
|
|
||||||
use OCP\UserMigration\UserMigrationException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Per-user backup via user_migration: every registered migrator's export streams straight into
|
|
||||||
* the encrypted repository (no zip, no copy in the user's own storage, see
|
|
||||||
* RepositoryExportDestination). Shares blobs with the full backup; prune() keeps a user export's
|
|
||||||
* blobs referenced through its manifest under users/<uid>/.
|
|
||||||
*
|
|
||||||
* user_migration is an optional dependency (unlike files_external): without it, only this
|
|
||||||
* per-user backup/restore is unavailable, the full backup still works.
|
|
||||||
*/
|
|
||||||
final class UserBackupService {
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private LeaseService $leases,
|
|
||||||
private UserMigrationServiceLocator $locator,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @throws UserMigrationException when user_migration is not enabled, or the export itself fails */
|
|
||||||
public function backupUser(Target $target, IUser $user): string {
|
|
||||||
$service = $this->locator->get();
|
|
||||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $user, $service) {
|
|
||||||
$repo = $this->targets->repository($target);
|
|
||||||
$dest = new RepositoryExportDestination($repo, $user->getUID(), LeaseService::throttled($refresh));
|
|
||||||
$service->export($dest, $user);
|
|
||||||
// A short exclusive lock around the catalog write only: the shared lock above
|
|
||||||
// allows another concurrent backup (full or per-user) to be writing the catalog at
|
|
||||||
// the same time. The repository is reopened inside the lock, so the write builds on
|
|
||||||
// the latest catalog generation instead of the one loaded before the lock was taken.
|
|
||||||
$this->withCatalogLock($target, fn () => $this->targets->repository($target)->recordUserExport($dest->manifestPath()));
|
|
||||||
return $dest->manifestPath();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private function withSharedLock(Target $target, callable $fn): mixed {
|
|
||||||
try {
|
|
||||||
// Refreshed by RepositoryExportDestination as it writes data, so a long export keeps it.
|
|
||||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 3600, $fn);
|
|
||||||
} catch (LockedException) {
|
|
||||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function withCatalogLock(Target $target, callable $fn): mixed {
|
|
||||||
try {
|
|
||||||
return $this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300, $fn);
|
|
||||||
} catch (LockedException) {
|
|
||||||
throw new \RuntimeException('Another catalog write for ' . $target->getName() . ' is in progress; try again');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,137 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Db\Target;
|
|
||||||
use OCA\NgBackup\UserMigration\RepositoryExportDestination;
|
|
||||||
use OCA\NgBackup\UserMigration\RepositoryImportSource;
|
|
||||||
use OCA\NgBackup\UserMigration\UserMigrationServiceLocator;
|
|
||||||
use OCP\EventDispatcher\IEventDispatcher;
|
|
||||||
use OCP\Files\Events\Node\FilesystemTornDownEvent;
|
|
||||||
use OCP\IUser;
|
|
||||||
use OCP\IUserManager;
|
|
||||||
use OCP\Lock\LockedException;
|
|
||||||
use OCP\Security\ISecureRandom;
|
|
||||||
use OCP\UserMigration\UserMigrationException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Restoring a user from a user_migration export (see UserBackupService), never over an existing
|
|
||||||
* account:
|
|
||||||
* - replace: a safety export of the current state goes into the repository first, then the
|
|
||||||
* account is deleted and recreated fresh from the backup, under the same uid;
|
|
||||||
* - as-backup: the backup is restored into a new account "<uid>-bak" (or "-bak-2", ...) instead,
|
|
||||||
* so an admin can compare and transfer by hand. Nextcloud cannot rename a uid, so the *backup*
|
|
||||||
* gets the new name, never the existing account.
|
|
||||||
*/
|
|
||||||
final class UserRestoreService {
|
|
||||||
public const MODE_REPLACE = 'replace';
|
|
||||||
public const MODE_AS_BACKUP = 'as-backup';
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private LeaseService $leases,
|
|
||||||
private UserMigrationServiceLocator $locator,
|
|
||||||
private IUserManager $userManager,
|
|
||||||
private ISecureRandom $random,
|
|
||||||
private IEventDispatcher $events,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @throws UserMigrationException when user_migration is not enabled, or the restore itself fails */
|
|
||||||
public function restoreUser(Target $target, string $manifestPath, string $mode = self::MODE_REPLACE): IUser {
|
|
||||||
$service = $this->locator->get();
|
|
||||||
return $this->withSharedLock($target, function (callable $refresh) use ($target, $manifestPath, $mode, $service): IUser {
|
|
||||||
$repo = $this->targets->repository($target);
|
|
||||||
// Only ever read a manifest the verified catalog actually recorded: a path alone
|
|
||||||
// (even one this app generated before) proves nothing about it still being current
|
|
||||||
// or genuine once the catalog's rollback protection is supposed to vouch for it.
|
|
||||||
if (!$repo->isRecordedUserExport($manifestPath)) {
|
|
||||||
throw new \InvalidArgumentException("$manifestPath is not a recorded user export for this location");
|
|
||||||
}
|
|
||||||
$beat = LeaseService::throttled($refresh);
|
|
||||||
$source = new RepositoryImportSource($repo, $manifestPath, $beat);
|
|
||||||
$originalUid = $source->getOriginalUid();
|
|
||||||
|
|
||||||
if ($mode === self::MODE_AS_BACKUP) {
|
|
||||||
$backupUid = $this->freeBackupUid($originalUid);
|
|
||||||
$user = $this->userManager->createUser($backupUid, $this->random->generate(32));
|
|
||||||
if ($user === false) {
|
|
||||||
throw new \RuntimeException("Could not create the account $backupUid");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return $service->import($source, $user);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
// A partially-imported, never-used account left behind is strictly worse
|
|
||||||
// than no account at all; the original source account was never touched.
|
|
||||||
$user->delete();
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($mode !== self::MODE_REPLACE) {
|
|
||||||
throw new \InvalidArgumentException("Unknown restore mode: $mode");
|
|
||||||
}
|
|
||||||
$existing = $this->userManager->get($originalUid);
|
|
||||||
$safetyManifestPath = null;
|
|
||||||
if ($existing !== null) {
|
|
||||||
// Never import over a live account: export its current state first, so nothing
|
|
||||||
// is lost if the restore turns out to be the wrong choice.
|
|
||||||
$safety = new RepositoryExportDestination($repo, $originalUid, $beat);
|
|
||||||
$service->export($safety, $existing);
|
|
||||||
$safetyManifestPath = $safety->manifestPath();
|
|
||||||
// Same race as UserBackupService::backupUser(): reopen the repository inside the
|
|
||||||
// catalog lock so the write builds on the latest generation.
|
|
||||||
$this->withCatalogLock($target, fn () => $this->targets->repository($target)->recordUserExport($safetyManifestPath));
|
|
||||||
if (!$existing->delete()) {
|
|
||||||
throw new \RuntimeException("Could not delete the existing account $originalUid before restoring; its safety export is at $safetyManifestPath");
|
|
||||||
}
|
|
||||||
// IRootFolder caches a Folder per uid for the lifetime of this process; without
|
|
||||||
// this it would still hand the just-deleted account's (now storage-less) folder
|
|
||||||
// to the import below, and every write would fail.
|
|
||||||
$this->events->dispatchTyped(new FilesystemTornDownEvent());
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return $service->import($source, null);
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
// The original account is already gone at this point (or never existed); a
|
|
||||||
// failure here can only be reported, not undone in place. Its safety export
|
|
||||||
// (if any) is the recovery path, already recorded above.
|
|
||||||
throw new UserMigrationException(
|
|
||||||
"Could not restore $originalUid" . ($safetyManifestPath !== null ? "; its safety export is at $safetyManifestPath" : '') . ': ' . $e->getMessage(),
|
|
||||||
0,
|
|
||||||
$e,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private function freeBackupUid(string $uid): string {
|
|
||||||
$candidate = $uid . '-bak';
|
|
||||||
for ($n = 2; $this->userManager->userExists($candidate); $n++) {
|
|
||||||
$candidate = $uid . '-bak-' . $n;
|
|
||||||
}
|
|
||||||
return $candidate;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function withSharedLock(Target $target, callable $fn): mixed {
|
|
||||||
try {
|
|
||||||
// Refreshed while the safety export writes and the import reads (see UserBackupService::withSharedLock()).
|
|
||||||
return $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, 3600, $fn);
|
|
||||||
} catch (LockedException) {
|
|
||||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function withCatalogLock(Target $target, callable $fn): mixed {
|
|
||||||
try {
|
|
||||||
return $this->leases->with(PruneService::catalogLockKey($target), LeaseService::EXCLUSIVE, 300, $fn);
|
|
||||||
} catch (LockedException) {
|
|
||||||
throw new \RuntimeException('Another catalog write for ' . $target->getName() . ' is in progress; try again');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,158 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\Service;
|
|
||||||
|
|
||||||
use OCA\NgBackup\AppInfo\Application;
|
|
||||||
use OCA\NgBackup\Db\Target;
|
|
||||||
use OCP\IAppConfig;
|
|
||||||
use OCP\Lock\LockedException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checksum audit of a snapshot (occ backup:verify), see Repository::verify(). On a large
|
|
||||||
* repository a check takes hours (a deep one days), so how far it is gets stored every
|
|
||||||
* PROGRESS_SECONDS, together with how it ended: `occ backup:status` shows it.
|
|
||||||
*/
|
|
||||||
final class VerifyService {
|
|
||||||
public const PROGRESS_SECONDS = 30;
|
|
||||||
private const KEY_PREFIX = 'verify_progress_';
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private TargetService $targets,
|
|
||||||
private LeaseService $leases,
|
|
||||||
private IAppConfig $appConfig,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param ?callable(array):void $onProgress called with the stored progress every PROGRESS_SECONDS
|
|
||||||
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
|
||||||
*/
|
|
||||||
public function verify(Target $target, string $snapshotId, bool $deep = false, ?callable $onProgress = null): array {
|
|
||||||
return $this->run($target, $snapshotId, $deep, $onProgress, fn (\OCA\NgBackup\Repository\Repository $repo, callable $heartbeat): array => $repo->verify($snapshotId, $deep, $heartbeat));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param ?callable(array):void $onProgress called with the stored progress every PROGRESS_SECONDS
|
|
||||||
* @return array{filesChecked:int, blobsChecked:int, bytesChecked:int, missing:list<string>, failed:list<string>, unreadable:list<string>, readError:string}
|
|
||||||
*/
|
|
||||||
public function verifyUserExport(Target $target, string $manifestPath, bool $deep = false, ?callable $onProgress = null): array {
|
|
||||||
return $this->run($target, $manifestPath, $deep, $onProgress, fn (\OCA\NgBackup\Repository\Repository $repo, callable $heartbeat): array => $repo->verifyUserExport($manifestPath, $deep, $heartbeat));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** One line on a check's progress or outcome, for occ (see progress()). */
|
|
||||||
public static function describe(array $s): string {
|
|
||||||
$size = static fn (int $b): string => $b >= 1073741824 ? sprintf('%.1f GiB', (float)$b / 1073741824.0) : sprintf('%.1f MiB', (float)$b / 1048576.0);
|
|
||||||
$count = static fn (int $done, mixed $total): string => number_format($done)
|
|
||||||
. (is_int($total) && $total > 0 ? '/' . number_format($total) . sprintf(' (%d%%)', intdiv(100 * $done, $total)) : '');
|
|
||||||
$deep = ($s['deep'] ?? false) === true;
|
|
||||||
$kind = $deep ? 'deep check' : 'check';
|
|
||||||
if (($s['status'] ?? '') === 'running') {
|
|
||||||
$phase = match ($s['phase'] ?? '') {
|
|
||||||
'index' => 'index files ' . $count((int)$s['done'], $s['total'] ?? null),
|
|
||||||
'packs' => 'pack folders ' . $count((int)$s['done'], $s['total'] ?? null),
|
|
||||||
'files', 'database' => 'files ' . $count((int)$s['done'], $s['total'] ?? null) . ', ' . number_format((int)($s['blobs'] ?? 0)) . ' blobs'
|
|
||||||
. ($deep ? ', ' . $size((int)($s['bytes'] ?? 0)) . ' decrypted' : ''),
|
|
||||||
default => 'starting',
|
|
||||||
};
|
|
||||||
$age = time() - (int)($s['updated'] ?? 0);
|
|
||||||
return sprintf('%s running since %s: %s; updated %s ago%s', $kind, date('Y-m-d H:i', (int)$s['started']), $phase,
|
|
||||||
$age < 120 ? $age . ' s' : intdiv($age, 60) . ' min',
|
|
||||||
$age > 600 ? ' (no progress for a while: the process may have stopped)' : '');
|
|
||||||
}
|
|
||||||
$took = gmdate('H:i:s', max(0, (int)($s['finished'] ?? 0) - (int)($s['started'] ?? 0)));
|
|
||||||
return match ($s['status'] ?? '') {
|
|
||||||
'ok' => sprintf('%s OK on %s (took %s): %s files, %s blobs%s', $kind, date('Y-m-d H:i', (int)$s['finished']), $took,
|
|
||||||
number_format((int)($s['files'] ?? 0)), number_format((int)($s['blobs'] ?? 0)), $deep ? ', ' . $size((int)($s['bytes'] ?? 0)) . ' decrypted' : ''),
|
|
||||||
'problems' => sprintf('%s found PROBLEMS on %s: %d missing, %d failed, %d unreadable blob(s); run occ backup:verify again for the details',
|
|
||||||
$kind, date('Y-m-d H:i', (int)$s['finished']), (int)($s['missing'] ?? 0), (int)($s['failedBlobs'] ?? 0), (int)($s['unreadable'] ?? 0)),
|
|
||||||
default => sprintf('%s FAILED on %s: %s', $kind, date('Y-m-d H:i', (int)($s['finished'] ?? 0)), (string)($s['error'] ?? 'unknown error')),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The stored state of the last check per location: running (with how far it is) or how it
|
|
||||||
* ended.
|
|
||||||
*
|
|
||||||
* @return array<int, array<string, mixed>> by location id
|
|
||||||
*/
|
|
||||||
public function progress(): array {
|
|
||||||
$result = [];
|
|
||||||
foreach ($this->appConfig->getKeys(Application::APP_ID) as $key) {
|
|
||||||
if (!str_starts_with($key, self::KEY_PREFIX)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$state = json_decode($this->appConfig->getValueString(Application::APP_ID, $key, '', lazy: true), true);
|
|
||||||
if (is_array($state)) {
|
|
||||||
$result[(int)substr($key, strlen(self::KEY_PREFIX))] = $state;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ksort($result);
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function run(Target $target, string $what, bool $deep, ?callable $onProgress, callable $fn): array {
|
|
||||||
$ttl = $deep ? 600 : 300;
|
|
||||||
$state = ['what' => $what, 'deep' => $deep, 'status' => 'running', 'started' => time(), 'updated' => time(),
|
|
||||||
'pid' => getmypid(), 'phase' => 'starting', 'done' => 0, 'total' => null, 'blobs' => 0, 'bytes' => 0];
|
|
||||||
try {
|
|
||||||
$this->store($target, $state);
|
|
||||||
} catch (\Throwable) {
|
|
||||||
// progress reports are a convenience; the check itself goes ahead
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$result = $this->leases->with(PruneService::lockKey($target), LeaseService::SHARED, $ttl, function (callable $refresh) use ($target, $ttl, $fn, $onProgress, &$state): array {
|
|
||||||
$lastRefresh = time();
|
|
||||||
$lastStore = time();
|
|
||||||
$heartbeat = function (?array $progress = null) use ($refresh, $ttl, $target, $onProgress, &$lastRefresh, &$lastStore, &$state): void {
|
|
||||||
$now = time();
|
|
||||||
if ($now - $lastRefresh >= intdiv($ttl, 2)) {
|
|
||||||
$refresh();
|
|
||||||
$lastRefresh = $now;
|
|
||||||
}
|
|
||||||
if ($progress !== null && $now - $lastStore >= self::PROGRESS_SECONDS) {
|
|
||||||
$state = array_merge($state, $progress, ['updated' => $now]);
|
|
||||||
try {
|
|
||||||
$this->store($target, $state);
|
|
||||||
} catch (\Throwable) {
|
|
||||||
// a progress report that cannot be stored must not stop the check
|
|
||||||
}
|
|
||||||
$lastStore = $now;
|
|
||||||
if ($onProgress !== null) {
|
|
||||||
$onProgress($state);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
return $fn($this->targets->repository($target), $heartbeat);
|
|
||||||
});
|
|
||||||
} catch (LockedException) {
|
|
||||||
$this->finish($target, $state, 'failed', ['error' => 'A cleanup of the location was in progress']);
|
|
||||||
throw new \RuntimeException('A cleanup of ' . $target->getName() . ' is in progress; try again later');
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
$this->finish($target, $state, 'failed', ['error' => $e->getMessage()]);
|
|
||||||
throw $e;
|
|
||||||
}
|
|
||||||
$problems = count($result['missing']) + count($result['failed']) + count($result['unreadable']);
|
|
||||||
$this->finish($target, $state, $problems === 0 ? 'ok' : 'problems', [
|
|
||||||
'phase' => 'done', 'files' => $result['filesChecked'], 'blobs' => $result['blobsChecked'], 'bytes' => $result['bytesChecked'],
|
|
||||||
'missing' => count($result['missing']), 'failedBlobs' => count($result['failed']), 'unreadable' => count($result['unreadable']),
|
|
||||||
]);
|
|
||||||
return $result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function finish(Target $target, array $state, string $status, array $extra): void {
|
|
||||||
try {
|
|
||||||
$this->store($target, array_merge($state, $extra, ['status' => $status, 'updated' => time(), 'finished' => time()]));
|
|
||||||
} catch (\Throwable) {
|
|
||||||
// the check's own result counts; its stored summary is a convenience
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function store(Target $target, array $state): void {
|
|
||||||
$this->appConfig->setValueString(Application::APP_ID, self::KEY_PREFIX . $target->getId(), json_encode($state, JSON_THROW_ON_ERROR), lazy: true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -11,7 +11,7 @@ use OCP\AppFramework\Http\TemplateResponse;
|
|||||||
use OCP\Settings\ISettings;
|
use OCP\Settings\ISettings;
|
||||||
use OCP\Util;
|
use OCP\Util;
|
||||||
|
|
||||||
final class Admin implements ISettings {
|
class Admin implements ISettings {
|
||||||
public function getForm(): TemplateResponse {
|
public function getForm(): TemplateResponse {
|
||||||
Util::addScript('ng_backup', 'ng_backup-admin');
|
Util::addScript('ng_backup', 'ng_backup-admin');
|
||||||
Util::addStyle('ng_backup', 'ng_backup-admin');
|
Util::addStyle('ng_backup', 'ng_backup-admin');
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use OCP\IL10N;
|
|||||||
use OCP\IURLGenerator;
|
use OCP\IURLGenerator;
|
||||||
use OCP\Settings\IIconSection;
|
use OCP\Settings\IIconSection;
|
||||||
|
|
||||||
final class AdminSection implements IIconSection {
|
class AdminSection implements IIconSection {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private IL10N $l,
|
private IL10N $l,
|
||||||
private IURLGenerator $url,
|
private IURLGenerator $url,
|
||||||
|
|||||||
@@ -1,71 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\SetupCheck;
|
|
||||||
|
|
||||||
use OCA\NgBackup\Db\Run;
|
|
||||||
use OCA\NgBackup\Db\RunMapper;
|
|
||||||
use OCA\NgBackup\Service\AlertService;
|
|
||||||
use OCA\NgBackup\Service\BackupService;
|
|
||||||
use OCA\NgBackup\Service\KeyService;
|
|
||||||
use OCA\NgBackup\Service\TargetService;
|
|
||||||
use OCP\IL10N;
|
|
||||||
use OCP\IURLGenerator;
|
|
||||||
use OCP\SetupCheck\ISetupCheck;
|
|
||||||
use OCP\SetupCheck\SetupResult;
|
|
||||||
|
|
||||||
/** Shows the backup state in the admin overview (Security & setup warnings). */
|
|
||||||
final class BackupStatusCheck implements ISetupCheck {
|
|
||||||
public function __construct(
|
|
||||||
private IL10N $l,
|
|
||||||
private IURLGenerator $url,
|
|
||||||
private KeyService $keys,
|
|
||||||
private TargetService $targets,
|
|
||||||
private RunMapper $runs,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getCategory(): string {
|
|
||||||
return 'system';
|
|
||||||
}
|
|
||||||
|
|
||||||
public function getName(): string {
|
|
||||||
return $this->l->t('NG Backup');
|
|
||||||
}
|
|
||||||
|
|
||||||
public function run(): SetupResult {
|
|
||||||
$link = $this->url->linkToRouteAbsolute('settings.AdminSettings.index', ['section' => 'ng_backup']);
|
|
||||||
if (!$this->keys->isInitialized() || !$this->keys->everConfirmed()) {
|
|
||||||
return SetupResult::error($this->l->t('Backups are not set up: create the backup key and confirm the recovery kit.'), $link);
|
|
||||||
}
|
|
||||||
if ($this->keys->recoveryKitConfirmation() === null) {
|
|
||||||
return SetupResult::error($this->l->t('The backup passphrase was changed: download and confirm the new recovery kit.'), $link);
|
|
||||||
}
|
|
||||||
$targets = $this->targets->list();
|
|
||||||
if ($targets === []) {
|
|
||||||
return SetupResult::warning($this->l->t('No backup location is configured.'), $link);
|
|
||||||
}
|
|
||||||
$problems = [];
|
|
||||||
foreach ($targets as $t) {
|
|
||||||
$last = $this->runs->lastFinished($t->getId(), BackupService::KIND_FULL);
|
|
||||||
if ($last === null) {
|
|
||||||
$problems[] = $this->l->t('%s: no successful backup yet', [$t->getName()]);
|
|
||||||
} elseif (time() - (int)$last->getFinishedAt() > AlertService::STALE_AFTER) {
|
|
||||||
$problems[] = $this->l->t('%1$s: last successful backup %2$s', [$t->getName(), date('Y-m-d H:i', (int)$last->getFinishedAt())]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
foreach ($this->runs->findRecent(10) as $r) {
|
|
||||||
if ($r->getStatus() === Run::FAILED && $r->getStartedAt() > time() - AlertService::STALE_AFTER) {
|
|
||||||
$problems[] = $this->l->t('A %1$s run failed: %2$s', [$r->getKind(), mb_strimwidth((string)$r->getError(), 0, 120, '…')]);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return $problems === []
|
|
||||||
? SetupResult::success($this->l->t('Recent backups to all locations succeeded.'))
|
|
||||||
: SetupResult::warning(implode("\n", $problems), $link);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -28,8 +28,6 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private Repository $repo,
|
private Repository $repo,
|
||||||
private string $uid,
|
private string $uid,
|
||||||
/** Called while data is written (per chunk), to keep the location lease alive. */
|
|
||||||
private ?\Closure $heartbeat = null,
|
|
||||||
) {
|
) {
|
||||||
$this->exportId = gmdate('Ymd\THis\Z') . '-' . bin2hex(random_bytes(4));
|
$this->exportId = gmdate('Ymd\THis\Z') . '-' . bin2hex(random_bytes(4));
|
||||||
}
|
}
|
||||||
@@ -38,7 +36,6 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
$writer = $this->repo->blobWriter($this->stats);
|
$writer = $this->repo->blobWriter($this->stats);
|
||||||
$writer->write($content);
|
$writer->write($content);
|
||||||
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => strlen($content)];
|
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => strlen($content)];
|
||||||
$this->beat();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function addFileAsStream(string $path, $stream): void {
|
public function addFileAsStream(string $path, $stream): void {
|
||||||
@@ -49,7 +46,6 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
throw new UserMigrationException('Read error while exporting ' . $path);
|
throw new UserMigrationException('Read error while exporting ' . $path);
|
||||||
}
|
}
|
||||||
$writer->write($chunk);
|
$writer->write($chunk);
|
||||||
$this->beat();
|
|
||||||
}
|
}
|
||||||
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => $writer->bytes()];
|
$this->entries[self::norm($path)] = ['t' => 'f', 'b' => $writer->finish(), 's' => $writer->bytes()];
|
||||||
}
|
}
|
||||||
@@ -69,11 +65,8 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
if ($stream === false) {
|
if ($stream === false) {
|
||||||
throw new \RuntimeException('Cannot read ' . $node->getPath());
|
throw new \RuntimeException('Cannot read ' . $node->getPath());
|
||||||
}
|
}
|
||||||
try {
|
$this->addFileAsStream($target, $stream);
|
||||||
$this->addFileAsStream($target, $stream);
|
fclose($stream);
|
||||||
} finally {
|
|
||||||
fclose($stream);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -96,12 +89,6 @@ final class RepositoryExportDestination implements IExportDestination {
|
|||||||
return $this->stats;
|
return $this->stats;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function beat(): void {
|
|
||||||
if ($this->heartbeat !== null) {
|
|
||||||
($this->heartbeat)();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static function norm(string $path): string {
|
private static function norm(string $path): string {
|
||||||
return trim($path, '/');
|
return trim($path, '/');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ declare(strict_types=1);
|
|||||||
namespace OCA\NgBackup\UserMigration;
|
namespace OCA\NgBackup\UserMigration;
|
||||||
|
|
||||||
use OCA\NgBackup\Repository\Repository;
|
use OCA\NgBackup\Repository\Repository;
|
||||||
use OCP\Files\File;
|
|
||||||
use OCP\Files\Folder;
|
use OCP\Files\Folder;
|
||||||
use OCP\UserMigration\IImportSource;
|
use OCP\UserMigration\IImportSource;
|
||||||
use OCP\UserMigration\UserMigrationException;
|
use OCP\UserMigration\UserMigrationException;
|
||||||
@@ -20,8 +19,6 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
public function __construct(
|
public function __construct(
|
||||||
private Repository $repo,
|
private Repository $repo,
|
||||||
string $manifestPath,
|
string $manifestPath,
|
||||||
/** Called while blobs are read, to keep the location lease alive. */
|
|
||||||
private ?\Closure $heartbeat = null,
|
|
||||||
) {
|
) {
|
||||||
$this->manifest = json_decode($repo->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
$this->manifest = json_decode($repo->getObject($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
||||||
}
|
}
|
||||||
@@ -33,18 +30,11 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
return $data;
|
return $data;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return resource */
|
|
||||||
public function getFileAsStream(string $path) {
|
public function getFileAsStream(string $path) {
|
||||||
$entry = $this->entry($path, 'f');
|
$entry = $this->entry($path, 'f');
|
||||||
// Phase 0: buffer in php://temp (memory up to 8 MiB, then a temp file for this one file).
|
// Phase 0: buffer in php://temp (memory up to 8 MiB, then a temp file for this one file).
|
||||||
$stream = fopen('php://temp/maxmemory:' . (8 * 1048576), 'w+b');
|
$stream = fopen('php://temp/maxmemory:' . (8 * 1048576), 'w+b');
|
||||||
if ($stream === false) {
|
|
||||||
throw new UserMigrationException('Cannot buffer ' . $path);
|
|
||||||
}
|
|
||||||
foreach ($this->repo->readBlobs($entry['b']) as $data) {
|
foreach ($this->repo->readBlobs($entry['b']) as $data) {
|
||||||
if ($this->heartbeat !== null) {
|
|
||||||
($this->heartbeat)();
|
|
||||||
}
|
|
||||||
for ($done = 0, $len = strlen($data); $done < $len; $done += $n) {
|
for ($done = 0, $len = strlen($data); $done < $len; $done += $n) {
|
||||||
$n = fwrite($stream, $done === 0 ? $data : substr($data, $done));
|
$n = fwrite($stream, $done === 0 ? $data : substr($data, $done));
|
||||||
if ($n === false || $n === 0) {
|
if ($n === false || $n === 0) {
|
||||||
@@ -61,7 +51,6 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
$prefix = trim($path, '/');
|
$prefix = trim($path, '/');
|
||||||
$prefix = $prefix === '' ? '' : $prefix . '/';
|
$prefix = $prefix === '' ? '' : $prefix . '/';
|
||||||
$names = [];
|
$names = [];
|
||||||
/** @var string $p */
|
|
||||||
foreach (array_keys($this->manifest['entries']) as $p) {
|
foreach (array_keys($this->manifest['entries']) as $p) {
|
||||||
if ($prefix === '' || str_starts_with($p, $prefix)) {
|
if ($prefix === '' || str_starts_with($p, $prefix)) {
|
||||||
$rest = substr($p, strlen($prefix));
|
$rest = substr($p, strlen($prefix));
|
||||||
@@ -81,7 +70,6 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
if (isset($this->manifest['entries'][$p])) {
|
if (isset($this->manifest['entries'][$p])) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
/** @var string $k */
|
|
||||||
foreach (array_keys($this->manifest['entries']) as $k) {
|
foreach (array_keys($this->manifest['entries']) as $k) {
|
||||||
if (str_starts_with($k, $p . '/')) {
|
if (str_starts_with($k, $p . '/')) {
|
||||||
return true;
|
return true;
|
||||||
@@ -103,25 +91,13 @@ final class RepositoryImportSource implements IImportSource {
|
|||||||
$this->copyToFolder($sub, $path);
|
$this->copyToFolder($sub, $path);
|
||||||
} else {
|
} else {
|
||||||
$stream = $this->getFileAsStream($path);
|
$stream = $this->getFileAsStream($path);
|
||||||
try {
|
if ($destination->nodeExists($name)) {
|
||||||
if ($destination->nodeExists($name)) {
|
$destination->get($name)->putContent($stream);
|
||||||
$existing = $destination->get($name);
|
} else {
|
||||||
if (!$existing instanceof File) {
|
$destination->newFile($name, $stream);
|
||||||
throw new UserMigrationException("$path exists and is not a file");
|
}
|
||||||
}
|
if (is_resource($stream)) {
|
||||||
$existing->putContent($stream);
|
fclose($stream);
|
||||||
} else {
|
|
||||||
$destination->newFile($name, $stream);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
// putContent()/newFile() already close the stream once they've consumed it:
|
|
||||||
// despite the @return resource docblock above, $stream is no longer a
|
|
||||||
// resource by the time we get here (verified against a real storage; a
|
|
||||||
// double fclose() throws a TypeError on PHP 8).
|
|
||||||
/** @psalm-suppress RedundantConditionGivenDocblockType */
|
|
||||||
if (is_resource($stream)) {
|
|
||||||
fclose($stream);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\UserMigration;
|
|
||||||
|
|
||||||
use OCP\App\AppPathNotFoundException;
|
|
||||||
use OCP\App\IAppManager;
|
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
|
||||||
use Symfony\Component\Console\Helper\QuestionHelper;
|
|
||||||
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Installs or enables the optional user_migration app, but only when the administrator asked for it
|
|
||||||
* (occ prompt or --install-user-migration). Runs occ in a child process, as the admin would.
|
|
||||||
*/
|
|
||||||
final class UserMigrationInstaller {
|
|
||||||
private const APP = 'user_migration';
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private IAppManager $appManager,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public function isEnabled(): bool {
|
|
||||||
return $this->appManager->isEnabledForAnyone(self::APP);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function codeIsPresent(): bool {
|
|
||||||
try {
|
|
||||||
$this->appManager->getAppPath(self::APP);
|
|
||||||
return true;
|
|
||||||
} catch (AppPathNotFoundException) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Installs (from the app store, if missing) and enables user_migration; throws on failure. */
|
|
||||||
public function install(): void {
|
|
||||||
$args = $this->codeIsPresent() ? ['app:enable', self::APP] : ['app:install', self::APP];
|
|
||||||
/** @psalm-suppress UndefinedClass */
|
|
||||||
$occ = \OC::$SERVERROOT . '/occ';
|
|
||||||
$command = implode(' ', array_map('escapeshellarg', [PHP_BINARY, $occ, ...$args])) . ' 2>&1';
|
|
||||||
exec($command, $lines, $code);
|
|
||||||
if ($code !== 0) {
|
|
||||||
throw new \RuntimeException("occ " . implode(' ', $args) . " failed:\n" . implode("\n", $lines));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns when user_migration is enabled. Otherwise asks the administrator (interactive shell) or
|
|
||||||
* requires --install-user-migration, and throws when neither gives consent.
|
|
||||||
*
|
|
||||||
* @throws \RuntimeException
|
|
||||||
*/
|
|
||||||
public function ensureEnabled(InputInterface $input, OutputInterface $output): void {
|
|
||||||
if ($this->isEnabled()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
$consent = $input->getOption('install-user-migration') === true;
|
|
||||||
if (!$consent && $input->isInteractive()) {
|
|
||||||
$consent = (new QuestionHelper())->ask($input, $output, new ConfirmationQuestion(
|
|
||||||
'The user_migration app is not enabled. Install and enable it now? [y/N] ', false)) === true;
|
|
||||||
}
|
|
||||||
if (!$consent) {
|
|
||||||
throw new \RuntimeException('The user_migration app is not enabled; per-user backup/restore needs it. Run "occ '
|
|
||||||
. 'app:install ' . self::APP . '" and "occ app:enable ' . self::APP . '", or repeat with --install-user-migration');
|
|
||||||
}
|
|
||||||
$output->writeln('Installing and enabling user_migration (approved by the administrator)...');
|
|
||||||
$this->install();
|
|
||||||
throw new \RuntimeException('user_migration is installed and enabled; run this command again');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
declare(strict_types=1);
|
|
||||||
|
|
||||||
// SPDX-FileCopyrightText: 2026 Dillard Blom
|
|
||||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
|
|
||||||
namespace OCA\NgBackup\UserMigration;
|
|
||||||
|
|
||||||
use OCP\App\IAppManager;
|
|
||||||
use OCP\Server;
|
|
||||||
use OCP\UserMigration\UserMigrationException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolves user_migration's own (internal, non-public) UserMigrationService on demand.
|
|
||||||
*
|
|
||||||
* Unlike files_external, user_migration is an optional dependency: without it, only per-user
|
|
||||||
* backup/restore is unavailable, the full backup still works. The class is never autoloaded by
|
|
||||||
* composer; everything that touches it is behind this one class.
|
|
||||||
*/
|
|
||||||
final class UserMigrationServiceLocator {
|
|
||||||
public function __construct(
|
|
||||||
private IAppManager $appManager,
|
|
||||||
) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/** @throws UserMigrationException when user_migration is not enabled anywhere */
|
|
||||||
public function get(): object {
|
|
||||||
if (!$this->appManager->isEnabledForAnyone('user_migration')) {
|
|
||||||
throw new UserMigrationException('The user_migration app is not enabled; per-user backup/restore is unavailable');
|
|
||||||
}
|
|
||||||
$this->appManager->loadApp('user_migration');
|
|
||||||
return Server::get(\OCA\UserMigration\Service\UserMigrationService::class);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Generated
-19
@@ -12,7 +12,6 @@
|
|||||||
"@nextcloud/axios": "^2.5.1",
|
"@nextcloud/axios": "^2.5.1",
|
||||||
"@nextcloud/dialogs": "^7.4.0",
|
"@nextcloud/dialogs": "^7.4.0",
|
||||||
"@nextcloud/l10n": "^3.4.0",
|
"@nextcloud/l10n": "^3.4.0",
|
||||||
"@nextcloud/password-confirmation": "^6.1.0",
|
|
||||||
"@nextcloud/router": "^3.0.1",
|
"@nextcloud/router": "^3.0.1",
|
||||||
"@nextcloud/vue": "^9.0.0",
|
"@nextcloud/vue": "^9.0.0",
|
||||||
"vue": "^3.5.0"
|
"vue": "^3.5.0"
|
||||||
@@ -886,24 +885,6 @@
|
|||||||
"node": "^20.0.0 || ^22.0.0 || ^24.0.0"
|
"node": "^20.0.0 || ^22.0.0 || ^24.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@nextcloud/password-confirmation": {
|
|
||||||
"version": "6.1.0",
|
|
||||||
"resolved": "https://registry.npmjs.org/@nextcloud/password-confirmation/-/password-confirmation-6.1.0.tgz",
|
|
||||||
"integrity": "sha512-N2ChXDbPcUcQCoAjj1yc65zfc61yiKpdM3qm/y3Y/y//NG7XWNNINwBg85lqJ2SISgmVStpsQ1d0blpFCoQXkQ==",
|
|
||||||
"license": "MIT",
|
|
||||||
"dependencies": {
|
|
||||||
"@nextcloud/auth": "^2.5.3",
|
|
||||||
"@nextcloud/axios": "^2.5.2",
|
|
||||||
"@nextcloud/l10n": "^3.4.1",
|
|
||||||
"@nextcloud/logger": "^3.0.3",
|
|
||||||
"@nextcloud/router": "^3.1.0",
|
|
||||||
"@nextcloud/vue": "^9.6.0",
|
|
||||||
"vue": "^3.5.30"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": "^20.0.0 || ^22.0.0 || ^24.0.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@nextcloud/paths": {
|
"node_modules/@nextcloud/paths": {
|
||||||
"version": "3.1.0",
|
"version": "3.1.0",
|
||||||
"resolved": "https://registry.npmjs.org/@nextcloud/paths/-/paths-3.1.0.tgz",
|
"resolved": "https://registry.npmjs.org/@nextcloud/paths/-/paths-3.1.0.tgz",
|
||||||
|
|||||||
@@ -12,7 +12,6 @@
|
|||||||
"@nextcloud/axios": "^2.5.1",
|
"@nextcloud/axios": "^2.5.1",
|
||||||
"@nextcloud/dialogs": "^7.4.0",
|
"@nextcloud/dialogs": "^7.4.0",
|
||||||
"@nextcloud/l10n": "^3.4.0",
|
"@nextcloud/l10n": "^3.4.0",
|
||||||
"@nextcloud/password-confirmation": "^6.1.0",
|
|
||||||
"@nextcloud/router": "^3.0.1",
|
"@nextcloud/router": "^3.0.1",
|
||||||
"@nextcloud/vue": "^9.0.0",
|
"@nextcloud/vue": "^9.0.0",
|
||||||
"vue": "^3.5.0"
|
"vue": "^3.5.0"
|
||||||
|
|||||||
@@ -1,484 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<files psalm-version="6.19.1@e2ca44251c1f1aa2e35452d426895746b47de64e">
|
|
||||||
<file src="lib/Backend/ExternalStorageFactory.php">
|
|
||||||
<MissingClosureParamType>
|
|
||||||
<code><![CDATA[$p]]></code>
|
|
||||||
<code><![CDATA[$p]]></code>
|
|
||||||
</MissingClosureParamType>
|
|
||||||
<MissingClosureReturnType>
|
|
||||||
<code><![CDATA[fn ($p) => $p->jsonSerialize()]]></code>
|
|
||||||
<code><![CDATA[fn ($p) => $p->jsonSerialize()]]></code>
|
|
||||||
</MissingClosureReturnType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Backend/IBackend.php">
|
|
||||||
<MissingParamType>
|
|
||||||
<code><![CDATA[$stream]]></code>
|
|
||||||
</MissingParamType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Backend/LocalBackend.php">
|
|
||||||
<MissingParamType>
|
|
||||||
<code><![CDATA[$stream]]></code>
|
|
||||||
</MissingParamType>
|
|
||||||
<TypeDoesNotContainType>
|
|
||||||
<code><![CDATA[$out !== false]]></code>
|
|
||||||
</TypeDoesNotContainType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Backend/StorageBackend.php">
|
|
||||||
<MissingParamType>
|
|
||||||
<code><![CDATA[$stream]]></code>
|
|
||||||
</MissingParamType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/BackgroundJob/BackupJob.php">
|
|
||||||
<MissingParamType>
|
|
||||||
<code><![CDATA[$argument]]></code>
|
|
||||||
</MissingParamType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/BackgroundJob/RestoreJob.php">
|
|
||||||
<MissingParamType>
|
|
||||||
<code><![CDATA[$argument]]></code>
|
|
||||||
</MissingParamType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/Browse.php">
|
|
||||||
<InvalidOperand>
|
|
||||||
<code><![CDATA[$b /= 1024]]></code>
|
|
||||||
</InvalidOperand>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/KeyConfirm.php">
|
|
||||||
<UndefinedInterfaceMethod>
|
|
||||||
<code><![CDATA[ask]]></code>
|
|
||||||
<code><![CDATA[ask]]></code>
|
|
||||||
</UndefinedInterfaceMethod>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/KeyImport.php">
|
|
||||||
<UndefinedInterfaceMethod>
|
|
||||||
<code><![CDATA[ask]]></code>
|
|
||||||
</UndefinedInterfaceMethod>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/KeyInit.php">
|
|
||||||
<UndefinedInterfaceMethod>
|
|
||||||
<code><![CDATA[ask]]></code>
|
|
||||||
<code><![CDATA[ask]]></code>
|
|
||||||
</UndefinedInterfaceMethod>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/KeySlots.php">
|
|
||||||
<UndefinedInterfaceMethod>
|
|
||||||
<code><![CDATA[ask]]></code>
|
|
||||||
<code><![CDATA[ask]]></code>
|
|
||||||
</UndefinedInterfaceMethod>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/Prune.php">
|
|
||||||
<PossiblyFalseOperand>
|
|
||||||
<code><![CDATA[json_encode($this->prune->policy()->toArray())]]></code>
|
|
||||||
</PossiblyFalseOperand>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/RunBackup.php">
|
|
||||||
<PossiblyNullArgument>
|
|
||||||
<code><![CDATA[$run->getSnapshotId()]]></code>
|
|
||||||
</PossiblyNullArgument>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/Status.php">
|
|
||||||
<RiskyTruthyFalsyComparison>
|
|
||||||
<code><![CDATA[$c]]></code>
|
|
||||||
<code><![CDATA[$r->getFinishedAt()]]></code>
|
|
||||||
</RiskyTruthyFalsyComparison>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/TargetAdd.php">
|
|
||||||
<PossiblyNullArgument>
|
|
||||||
<code><![CDATA[$r['target']->getRepositoryId()]]></code>
|
|
||||||
</PossiblyNullArgument>
|
|
||||||
<PossiblyUndefinedArrayOffset>
|
|
||||||
<code><![CDATA[$v]]></code>
|
|
||||||
</PossiblyUndefinedArrayOffset>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/TargetBackends.php">
|
|
||||||
<UndefinedClass>
|
|
||||||
<code><![CDATA[Base]]></code>
|
|
||||||
</UndefinedClass>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/TargetLimit.php">
|
|
||||||
<InvalidOperand>
|
|
||||||
<code><![CDATA[(float)$m[1] * $factor]]></code>
|
|
||||||
</InvalidOperand>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/TargetList.php">
|
|
||||||
<UndefinedClass>
|
|
||||||
<code><![CDATA[Base]]></code>
|
|
||||||
</UndefinedClass>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/TargetTrustCurrent.php">
|
|
||||||
<UndefinedInterfaceMethod>
|
|
||||||
<code><![CDATA[ask]]></code>
|
|
||||||
</UndefinedInterfaceMethod>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Command/Trash.php">
|
|
||||||
<RedundantCastGivenDocblockType>
|
|
||||||
<code><![CDATA[(string)($info['time'] ?? '')]]></code>
|
|
||||||
</RedundantCastGivenDocblockType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Controller/AdminApiController.php">
|
|
||||||
<InvalidOperand>
|
|
||||||
<code><![CDATA[$maxGb * 1073741824]]></code>
|
|
||||||
<code><![CDATA[$maxGb * 1073741824]]></code>
|
|
||||||
</InvalidOperand>
|
|
||||||
<PossiblyNullArgument>
|
|
||||||
<code><![CDATA[$r->getSnapshotId()]]></code>
|
|
||||||
</PossiblyNullArgument>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Crypto/StreamCipher.php">
|
|
||||||
<MissingParamType>
|
|
||||||
<code><![CDATA[$in]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
</MissingParamType>
|
|
||||||
<PossiblyFalseArgument>
|
|
||||||
<code><![CDATA[$in]]></code>
|
|
||||||
<code><![CDATA[$in]]></code>
|
|
||||||
<code><![CDATA[$in]]></code>
|
|
||||||
<code><![CDATA[$in]]></code>
|
|
||||||
<code><![CDATA[$in]]></code>
|
|
||||||
<code><![CDATA[$in]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
</PossiblyFalseArgument>
|
|
||||||
<PossiblyInvalidArrayAccess>
|
|
||||||
<code><![CDATA[unpack('N', $len)[1]]]></code>
|
|
||||||
</PossiblyInvalidArrayAccess>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Db/AppConfigCatalogAnchor.php">
|
|
||||||
<PossiblyFalseArgument>
|
|
||||||
<code><![CDATA[json_encode(['gen' => $gen, 'hash' => $hash])]]></code>
|
|
||||||
</PossiblyFalseArgument>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Db/DbDumper.php">
|
|
||||||
<InvalidArrayOffset>
|
|
||||||
<code><![CDATA[$info['sha256']]]></code>
|
|
||||||
<code><![CDATA[$manifest['sequences']]]></code>
|
|
||||||
</InvalidArrayOffset>
|
|
||||||
<MissingClosureReturnType>
|
|
||||||
<code><![CDATA[fn ($c) => $end[array_search($c, $columns, true)]]]></code>
|
|
||||||
<code><![CDATA[fn ($c) => $q($c)]]></code>
|
|
||||||
<code><![CDATA[fn (string $c) => $this->db->quoteIdentifier($c)]]></code>
|
|
||||||
<code><![CDATA[fn (string $c) => $this->db->quoteIdentifier($c)]]></code>
|
|
||||||
</MissingClosureReturnType>
|
|
||||||
<PossiblyFalseArgument>
|
|
||||||
<code><![CDATA[$value]]></code>
|
|
||||||
</PossiblyFalseArgument>
|
|
||||||
<PossiblyNullArgument>
|
|
||||||
<code><![CDATA[$columns]]></code>
|
|
||||||
</PossiblyNullArgument>
|
|
||||||
<UndefinedClass>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[$this->db]]></code>
|
|
||||||
<code><![CDATA[TransactionIsolationLevel]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\ParameterType]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\ParameterType]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\ParameterType]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\Platforms\PostgreSQLPlatform]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\Platforms\PostgreSQLPlatform]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\Platforms\PostgreSQLPlatform]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
|
||||||
<code><![CDATA[\Doctrine\DBAL\Schema\Table]]></code>
|
|
||||||
<code><![CDATA[private]]></code>
|
|
||||||
</UndefinedClass>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Db/Run.php">
|
|
||||||
<PropertyNotSetInConstructor>
|
|
||||||
<code><![CDATA[Run]]></code>
|
|
||||||
</PropertyNotSetInConstructor>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Db/Snapshot.php">
|
|
||||||
<PropertyNotSetInConstructor>
|
|
||||||
<code><![CDATA[Snapshot]]></code>
|
|
||||||
</PropertyNotSetInConstructor>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Db/Target.php">
|
|
||||||
<PropertyNotSetInConstructor>
|
|
||||||
<code><![CDATA[Target]]></code>
|
|
||||||
</PropertyNotSetInConstructor>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Job/BackupRun.php">
|
|
||||||
<ArgumentTypeCoercion>
|
|
||||||
<code><![CDATA[$blobs]]></code>
|
|
||||||
</ArgumentTypeCoercion>
|
|
||||||
<MissingParamType>
|
|
||||||
<code><![CDATA[$fh]]></code>
|
|
||||||
</MissingParamType>
|
|
||||||
<PossiblyUndefinedArrayOffset>
|
|
||||||
<code><![CDATA[$mtime]]></code>
|
|
||||||
<code><![CDATA[$state['phase']]]></code>
|
|
||||||
<code><![CDATA[$state['stats']]]></code>
|
|
||||||
</PossiblyUndefinedArrayOffset>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Job/ParentCursor.php">
|
|
||||||
<PossiblyNullReference>
|
|
||||||
<code><![CDATA[next]]></code>
|
|
||||||
</PossiblyNullReference>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Job/TreeWalker.php">
|
|
||||||
<RedundantCastGivenDocblockType>
|
|
||||||
<code><![CDATA[(string)$name]]></code>
|
|
||||||
</RedundantCastGivenDocblockType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Migration/Version000001Date20261002000000.php">
|
|
||||||
<UndefinedDocblockClass>
|
|
||||||
<code><![CDATA[$schema->createTable('ngb_targets')]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
</UndefinedDocblockClass>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Migration/Version000002Date20261002010000.php">
|
|
||||||
<UndefinedDocblockClass>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
</UndefinedDocblockClass>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Migration/Version000003Date20261002020000.php">
|
|
||||||
<UndefinedDocblockClass>
|
|
||||||
<code><![CDATA[$schema->getTable('ngb_targets')]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
</UndefinedDocblockClass>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Migration/Version000004Date20261002030000.php">
|
|
||||||
<UndefinedDocblockClass>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
</UndefinedDocblockClass>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Migration/Version000005Date20261002040000.php">
|
|
||||||
<UndefinedDocblockClass>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
<code><![CDATA[$t]]></code>
|
|
||||||
</UndefinedDocblockClass>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Repository/BlobIndex.php">
|
|
||||||
<PossiblyFalseArgument>
|
|
||||||
<code><![CDATA[$stream]]></code>
|
|
||||||
<code><![CDATA[$stream]]></code>
|
|
||||||
</PossiblyFalseArgument>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Repository/Catalog.php">
|
|
||||||
<PossiblyFalseArgument>
|
|
||||||
<code><![CDATA[$stream]]></code>
|
|
||||||
<code><![CDATA[$stream]]></code>
|
|
||||||
</PossiblyFalseArgument>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Repository/PackWriter.php">
|
|
||||||
<InvalidPropertyAssignmentValue>
|
|
||||||
<code><![CDATA[$this->buffer]]></code>
|
|
||||||
</InvalidPropertyAssignmentValue>
|
|
||||||
<PossiblyFalsePropertyAssignmentValue>
|
|
||||||
<code><![CDATA[fopen('php://temp/maxmemory:' . ($this->targetSize + 8 * 1048576), 'w+b')]]></code>
|
|
||||||
</PossiblyFalsePropertyAssignmentValue>
|
|
||||||
<RedundantPropertyInitializationCheck>
|
|
||||||
<code><![CDATA[isset($this->buffer)]]></code>
|
|
||||||
</RedundantPropertyInitializationCheck>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Repository/Repository.php">
|
|
||||||
<InvalidOperand>
|
|
||||||
<code><![CDATA[strlen($data) * 0.95]]></code>
|
|
||||||
</InvalidOperand>
|
|
||||||
<InvalidReturnStatement>
|
|
||||||
<code><![CDATA[['snapshot' => $snapshotId] + $stats]]></code>
|
|
||||||
</InvalidReturnStatement>
|
|
||||||
<InvalidReturnType>
|
|
||||||
<code><![CDATA[array{snapshot:string, files:int, reused:int, read:int, newBlobs:int, dupBlobs:int, uploaded:int}]]></code>
|
|
||||||
</InvalidReturnType>
|
|
||||||
<MissingParamType>
|
|
||||||
<code><![CDATA[$fh]]></code>
|
|
||||||
</MissingParamType>
|
|
||||||
<PossiblyFalseArgument>
|
|
||||||
<code><![CDATA[$encrypted]]></code>
|
|
||||||
<code><![CDATA[$encrypted]]></code>
|
|
||||||
<code><![CDATA[$encrypted]]></code>
|
|
||||||
<code><![CDATA[$encrypted]]></code>
|
|
||||||
<code><![CDATA[$fh]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$plain]]></code>
|
|
||||||
<code><![CDATA[$s]]></code>
|
|
||||||
<code><![CDATA[$s]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
<code><![CDATA[$tree]]></code>
|
|
||||||
</PossiblyFalseArgument>
|
|
||||||
<PossiblyUndefinedArrayOffset>
|
|
||||||
<code><![CDATA[$stats['files']]]></code>
|
|
||||||
<code><![CDATA[$stats['reused']]]></code>
|
|
||||||
</PossiblyUndefinedArrayOffset>
|
|
||||||
<PropertyNotSetInConstructor>
|
|
||||||
<code><![CDATA[$catalog]]></code>
|
|
||||||
</PropertyNotSetInConstructor>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Service/AlertService.php">
|
|
||||||
<InvalidOperand>
|
|
||||||
<code><![CDATA[$bytes / $limit * 100]]></code>
|
|
||||||
<code><![CDATA[0.8 * $limit]]></code>
|
|
||||||
</InvalidOperand>
|
|
||||||
<PossiblyFalseOperand>
|
|
||||||
<code><![CDATA[json_encode(array_diff_key($params, ['actor' => 1]), JSON_UNESCAPED_SLASHES)]]></code>
|
|
||||||
</PossiblyFalseOperand>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Service/BackupService.php">
|
|
||||||
<InvalidOperand>
|
|
||||||
<code><![CDATA[$deadline - microtime(true) + 600]]></code>
|
|
||||||
</InvalidOperand>
|
|
||||||
<UndefinedClass>
|
|
||||||
<code><![CDATA[\OC]]></code>
|
|
||||||
<code><![CDATA[\OC]]></code>
|
|
||||||
</UndefinedClass>
|
|
||||||
<UndefinedInterfaceMethod>
|
|
||||||
<code><![CDATA[getInner]]></code>
|
|
||||||
</UndefinedInterfaceMethod>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Service/KeyService.php">
|
|
||||||
<ArgumentTypeCoercion>
|
|
||||||
<code><![CDATA[array_diff(range(1, self::MAX_SLOTS), $used)]]></code>
|
|
||||||
</ArgumentTypeCoercion>
|
|
||||||
<LessSpecificReturnStatement>
|
|
||||||
<code><![CDATA[array_map(fn ($s) => ['slot' => $s['slot'], 'label' => $s['label'], 'created' => $s['created']], $this->wrappedKey()['slots'])]]></code>
|
|
||||||
</LessSpecificReturnStatement>
|
|
||||||
<MissingClosureParamType>
|
|
||||||
<code><![CDATA[$s]]></code>
|
|
||||||
<code><![CDATA[$s]]></code>
|
|
||||||
</MissingClosureParamType>
|
|
||||||
<MissingClosureReturnType>
|
|
||||||
<code><![CDATA[static fn (string $c) => strtoupper(preg_replace('/[^A-Za-z0-9]/', '', $c) ?? '')]]></code>
|
|
||||||
</MissingClosureReturnType>
|
|
||||||
<MoreSpecificReturnType>
|
|
||||||
<code><![CDATA[list<array{slot:int, label:string, created:int}>]]></code>
|
|
||||||
</MoreSpecificReturnType>
|
|
||||||
<PossiblyFalseArgument>
|
|
||||||
<code><![CDATA[base64_decode($this->crypto->decrypt($enc), true)]]></code>
|
|
||||||
</PossiblyFalseArgument>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Service/PruneService.php">
|
|
||||||
<LessSpecificReturnStatement>
|
|
||||||
<code><![CDATA[['kept' => $keep, 'forgotten' => $forget, 'purged' => $purged, 'prune' => $prune]]]></code>
|
|
||||||
</LessSpecificReturnStatement>
|
|
||||||
<MissingClosureReturnType>
|
|
||||||
<code><![CDATA[fn () => $fn()]]></code>
|
|
||||||
</MissingClosureReturnType>
|
|
||||||
<MoreSpecificReturnType>
|
|
||||||
<code><![CDATA[array{kept: array<string, list<string>>, forgotten: list<string>, purged: list<string>, prune: array}]]></code>
|
|
||||||
</MoreSpecificReturnType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Service/RestoreFullService.php">
|
|
||||||
<EmptyArrayAccess>
|
|
||||||
<code><![CDATA[$backupConfig[$key]]]></code>
|
|
||||||
</EmptyArrayAccess>
|
|
||||||
<UndefinedInterfaceMethod>
|
|
||||||
<code><![CDATA[getInner]]></code>
|
|
||||||
</UndefinedInterfaceMethod>
|
|
||||||
<UnresolvableInclude>
|
|
||||||
<code><![CDATA[include $path]]></code>
|
|
||||||
</UnresolvableInclude>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Service/RestoreService.php">
|
|
||||||
<LessSpecificReturnStatement>
|
|
||||||
<code><![CDATA[array_values($children)]]></code>
|
|
||||||
</LessSpecificReturnStatement>
|
|
||||||
<MissingDependency>
|
|
||||||
<code><![CDATA[$this->root]]></code>
|
|
||||||
<code><![CDATA[private]]></code>
|
|
||||||
</MissingDependency>
|
|
||||||
<MoreSpecificReturnType>
|
|
||||||
<code><![CDATA[list<array{name:string, type:string, size:int, mtime:int|null, files:int}>]]></code>
|
|
||||||
</MoreSpecificReturnType>
|
|
||||||
<PossiblyFalseArgument>
|
|
||||||
<code><![CDATA[$in]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
<code><![CDATA[$out]]></code>
|
|
||||||
</PossiblyFalseArgument>
|
|
||||||
<PossiblyNullArrayOffset>
|
|
||||||
<code><![CDATA[$children]]></code>
|
|
||||||
<code><![CDATA[$children]]></code>
|
|
||||||
</PossiblyNullArrayOffset>
|
|
||||||
<RedundantCastGivenDocblockType>
|
|
||||||
<code><![CDATA[(string)$relPath]]></code>
|
|
||||||
<code><![CDATA[(string)$relPath]]></code>
|
|
||||||
</RedundantCastGivenDocblockType>
|
|
||||||
</file>
|
|
||||||
<file src="lib/Service/TargetService.php">
|
|
||||||
<UndefinedDocblockClass>
|
|
||||||
<code><![CDATA[Server::get(\OCA\Files_External\Service\BackendService::class)]]></code>
|
|
||||||
</UndefinedDocblockClass>
|
|
||||||
<UnresolvableInclude>
|
|
||||||
<code><![CDATA[require_once $path . '/composer/autoload.php']]></code>
|
|
||||||
</UnresolvableInclude>
|
|
||||||
</file>
|
|
||||||
</files>
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user