Empty folders in backup and restore, a full disk stops a restore, progress in the Bin
- Record folders without files in the meta record of a snapshot and restore them (#27) - Errors writing to the restore directory stop the restore, also with --salvage - Resume: check the folder before skipping, clean temporaries in folders that are complete - Clean only read-ahead folders of our own; refresh the lease on skipped files - Drop tables a failed database restore created; keep --salvage from calling a restored database fresh - Retry a pack that fails to open on the storage (SMB) - Without cron, running backups take turns from the open page - Bin: read in parts with a counter and progress bar, show read errors, refresh when a run ends
This commit is contained in:
@@ -102,7 +102,7 @@ ok "key, kit and location on the origin"
|
||||
|
||||
NC_PASS=$(openssl rand -hex 16)
|
||||
docker exec -u www-data -e OC_PASS="$NC_PASS" ngb-frt-origin php occ user:add --password-from-env frtest >/dev/null
|
||||
docker exec ngb-frt-origin sh -c "mkdir -p /var/www/html/data/frtest/files/Docs && head -c 200000 /dev/urandom > /var/www/html/data/frtest/files/Docs/report.bin && echo 'hello disaster recovery' > /var/www/html/data/frtest/files/Docs/note.txt && chown -R www-data:www-data /var/www/html/data/frtest"
|
||||
docker exec ngb-frt-origin sh -c "mkdir -p /var/www/html/data/frtest/files/Docs /var/www/html/data/frtest/files/Empty/Nested && head -c 200000 /dev/urandom > /var/www/html/data/frtest/files/Docs/report.bin && echo 'hello disaster recovery' > /var/www/html/data/frtest/files/Docs/note.txt && chown -R www-data:www-data /var/www/html/data/frtest"
|
||||
if [ "$MODE" = sftp ]; then
|
||||
# about three packs of 32 MiB
|
||||
docker exec ngb-frt-origin sh -c "head -c 90000000 /dev/urandom > /var/www/html/data/frtest/files/Docs/large.bin && chown www-data:www-data /var/www/html/data/frtest/files/Docs/large.bin"
|
||||
@@ -214,6 +214,7 @@ if [ "$MODE" = salvage ]; then
|
||||
printf '%s\n' "$OUT" | grep "$SALVAGE" >/dev/null || { printf '%s\n' "$OUT" | tail -30; fail "the output does not name the salvage folder"; }
|
||||
[ "$(docker exec ngb-frt-restore cat "$SALVAGE/data/frtest/files/Docs/note.txt")" = "hello disaster recovery" ] || fail "the user's file is not in the salvage folder"
|
||||
docker exec ngb-frt-restore test -f "$SALVAGE/config/config.php" || fail "config.php is not in the salvage folder"
|
||||
docker exec ngb-frt-restore test -d "$SALVAGE/data/frtest/files/Empty/Nested" || fail "the empty folder is not in the salvage folder"
|
||||
run_on ngb-frt-restore user:list | grep frtest >/dev/null && fail "the fresh database was changed (the backed-up user is there)"
|
||||
run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode is still on after the salvage"
|
||||
run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after the salvage"
|
||||
@@ -254,8 +255,8 @@ if [ "$MODE" = salvage-files ]; then
|
||||
if (!file_exists($root . $p)) { $bad[] = "not listed and not there: $p"; continue; }
|
||||
hash_file("sha256", $root . $p) === $sum ? $same++ : $bad[] = "different: $p";
|
||||
}
|
||||
$temps = glob($root . "*/*.ngb-restore-*") ?: [];
|
||||
foreach ($temps as $t) { $bad[] = "temporary left: $t"; }
|
||||
$walk = new RecursiveIteratorIterator(new RecursiveDirectoryIterator("/var/www/html/data", FilesystemIterator::SKIP_DOTS), RecursiveIteratorIterator::SELF_FIRST);
|
||||
foreach ($walk as $f) { if (str_contains($f->getFilename(), ".ngb-restore-")) { $bad[] = "temporary left: " . $f->getPathname(); } }
|
||||
echo "same=$same gone=$gone bad=", count($bad), "\n", implode("\n", array_slice($bad, 0, 10)), "\n";
|
||||
echo "reasons: ", implode(" | ", array_unique(array_map(fn ($r) => substr(preg_replace("#[0-9a-f]{16,}#", "<id>", $r), 0, 70), $reasons))), "\n";')
|
||||
printf '%s\n' "$CHECK"
|
||||
@@ -265,6 +266,8 @@ if [ "$MODE" = salvage-files ]; then
|
||||
ok "with --salvage: every file listed and absent, or there and identical"
|
||||
run_on ngb-frt-restore status 2>&1 | grep "maintenance: false" >/dev/null || fail "maintenance mode is still on after the salvage"
|
||||
run_on ngb-frt-restore user:list | grep frtest >/dev/null || fail "the backed-up user is missing: the database was not restored"
|
||||
run_on ngb-frt-restore user:list | grep admin >/dev/null || fail "the admin is missing: the database was not restored"
|
||||
docker exec ngb-frt-restore test -d /var/www/html/data/frtest/files/Empty/Nested || fail "the empty folder did not come back with --salvage"
|
||||
[ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt 2>/dev/null)" = "hello disaster recovery" ] \
|
||||
|| printf '%s\n' "$OUT" | grep "Docs/note.txt" >/dev/null || fail "note.txt is neither restored nor listed"
|
||||
run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails after the salvage"
|
||||
@@ -323,7 +326,10 @@ run_on ngb-frt-restore backup:status >/dev/null || fail "backup:status fails aft
|
||||
run_on ngb-frt-restore user:list | grep frtest >/dev/null || fail "the backed-up user is missing from the restored database"
|
||||
docker exec ngb-frt-restore test -f /var/www/html/data/frtest/files/Docs/note.txt || fail "restored file is missing"
|
||||
[ "$(docker exec ngb-frt-restore cat /var/www/html/data/frtest/files/Docs/note.txt)" = "hello disaster recovery" ] || fail "restored file content differs"
|
||||
docker exec ngb-frt-restore test -d /var/www/html/data/frtest/files/Empty/Nested || fail "the empty folder did not come back"
|
||||
run_on ngb-frt-restore files:scan frtest >/dev/null || fail "files:scan fails after restore"
|
||||
docker exec ngb-frt-restore test -d /var/www/html/data/frtest/files/Empty/Nested || fail "files:scan removed the empty folder"
|
||||
ok "the empty folder came back and stays after files:scan"
|
||||
# New rows after the restore: a new file (file cache) and a new user must not collide with
|
||||
# restored rows (ID counters after the data).
|
||||
docker exec ngb-frt-restore sh -c "echo new > /var/www/html/data/frtest/files/Docs/after-restore.txt && chown www-data /var/www/html/data/frtest/files/Docs/after-restore.txt"
|
||||
|
||||
@@ -26,7 +26,7 @@ ok "location added"
|
||||
# Test data: a user with files.
|
||||
$X sh -c "OC_PASS='Roundtrip-pw-2026' php occ user:add --password-from-env $U >/dev/null"
|
||||
# large.bin (40 MB) fills more than one pack: a restore to a directory reads ahead over several ranges
|
||||
$X sh -c "mkdir -p $DATA/$U/files/Docs/Sub && for i in 1 2 3 4 5; do head -c 300000 /dev/urandom > $DATA/$U/files/Docs/f\$i.bin; done; echo hello > $DATA/$U/files/Docs/Sub/note.txt; head -c 6000000 /dev/urandom > $DATA/$U/files/big.bin; head -c 40000000 /dev/urandom > $DATA/$U/files/large.bin"
|
||||
$X sh -c "mkdir -p $DATA/$U/files/Docs/Sub $DATA/$U/files/Docs/Empty/Nested $DATA/$U/files/Hollow && for i in 1 2 3 4 5; do head -c 300000 /dev/urandom > $DATA/$U/files/Docs/f\$i.bin; done; echo hello > $DATA/$U/files/Docs/Sub/note.txt; head -c 6000000 /dev/urandom > $DATA/$U/files/big.bin; head -c 40000000 /dev/urandom > $DATA/$U/files/large.bin"
|
||||
occ files:scan "$U" >/dev/null
|
||||
SUM_BEFORE=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
|
||||
|
||||
@@ -39,11 +39,17 @@ SNAP=$(occ backup:list ci | awk -F'|' '!f && / second /{gsub(/ /,"",$3); print $
|
||||
[ -n "$SNAP" ] || fail "snapshot not listed"
|
||||
|
||||
# Damage, then restore in each mode.
|
||||
$X sh -c "cd $DATA/$U/files && rm Docs/Sub/note.txt && echo broken > Docs/f1.bin && echo extra > Docs/extra.txt"
|
||||
$X sh -c "cd $DATA/$U/files && rm Docs/Sub/note.txt && echo broken > Docs/f1.bin && echo extra > Docs/extra.txt && rm -r Docs/Empty && rmdir Hollow"
|
||||
occ files:scan "$U" >/dev/null
|
||||
occ backup:restore ci "$SNAP" "data/$U/files/Docs" -m new-folder >/dev/null || fail "restore new-folder"
|
||||
$X sh -c "ls $DATA/$U/files | grep -q '^Restored '" || fail "no Restored folder"
|
||||
$X sh -c "test -d $DATA/$U/files/Restored\ */Docs/Empty/Nested" || fail "the empty folder is not in the Restored folder"
|
||||
occ backup:browse ci "$SNAP" "data/$U/files" | grep Hollow >/dev/null || fail "an empty folder is not shown when browsing the snapshot"
|
||||
occ backup:restore ci "$SNAP" "data/$U/files/Hollow" -m merge >/dev/null || fail "restore of a folder that was empty"
|
||||
$X sh -c "test -d $DATA/$U/files/Hollow" || fail "the empty folder did not come back by itself"
|
||||
occ backup:restore ci "$SNAP" "data/$U/files/Docs" -m replace >/dev/null || fail "restore replace"
|
||||
$X sh -c "test -d $DATA/$U/files/Docs/Empty/Nested" || fail "replace did not bring back the empty folder, or moved it to the trash"
|
||||
ok "empty folders: restored in each mode, by themselves and shown when browsing"
|
||||
$X sh -c "rm -rf '$DATA/$U/files/Restored '*" ; occ files:scan "$U" >/dev/null
|
||||
SUM_AFTER=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
|
||||
[ "$SUM_BEFORE" = "$SUM_AFTER" ] || fail "files differ after restore"
|
||||
@@ -63,6 +69,7 @@ occ backup:restore ci "$SNAP" "data/$U/files" -d /tmp/ngb-rt-dir >/dev/null || f
|
||||
SUM_DIR=$($X sh -c "cd /tmp/ngb-rt-dir/data/$U/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
|
||||
[ "$SUM_BEFORE" = "$SUM_DIR" ] || fail "the files restored to a directory differ"
|
||||
$X sh -c "ls -A /tmp/ngb-rt-dir | grep -q ngb-restore-spool" && fail "the read-ahead spool was left in the directory"
|
||||
$X sh -c "test -d /tmp/ngb-rt-dir/data/$U/files/Docs/Empty/Nested" || fail "the empty folder is not in the directory"
|
||||
$X sh -c "rm -rf /tmp/ngb-rt-dir"
|
||||
ok "restore of all the user's files to a directory: identical, no spool left"
|
||||
# not grep -q: it stops reading at the first match, and occ, still writing, then fails (pipefail)
|
||||
@@ -90,6 +97,9 @@ done
|
||||
[ "$RS" = done ] || fail "the backup started through the API ended as '$RS' after $i ticks"
|
||||
occ backup:list ci | grep " page " >/dev/null || fail "the snapshot of the ticked backup is not listed"
|
||||
ok "without system cron: warning in backup:status and the API, backup finished by $i ticks of the open page"
|
||||
# The Bin reads in parts, as the page does (what is in the parts: CatalogTest)
|
||||
api "$API/targets/$TID/trash?offset=0" | php -r '$p = json_decode(stream_get_contents(STDIN), true); exit(is_int($p["total"] ?? null) && is_array($p["items"] ?? null) && count($p["items"]) <= $p["total"] ? 0 : 1);' || fail "the Bin API does not give a total and items"
|
||||
ok "the Bin API reads in parts, with a total"
|
||||
occ background:cron >/dev/null
|
||||
$X php -f /var/www/html/cron.php >/dev/null 2>&1 || true
|
||||
occ backup:status | grep "system cron" >/dev/null && fail "backup:status still warns after cron ran"
|
||||
|
||||
Reference in New Issue
Block a user