Empty folders in backup and restore, a full disk stops a restore, progress in the Bin

- Record folders without files in the meta record of a snapshot and restore them (#27)
- Errors writing to the restore directory stop the restore, also with --salvage
- Resume: check the folder before skipping, clean temporaries in folders that are complete
- Clean only read-ahead folders of our own; refresh the lease on skipped files
- Drop tables a failed database restore created; keep --salvage from calling a restored database fresh
- Retry a pack that fails to open on the storage (SMB)
- Without cron, running backups take turns from the open page
- Bin: read in parts with a counter and progress bar, show read errors, refresh when a run ends

tests/integration/roundtrip.sh

@@ -26,7 +26,7 @@ ok "location added"
 # Test data: a user with files.
 $X sh -c "OC_PASS='Roundtrip-pw-2026' php occ user:add --password-from-env $U >/dev/null"
 # large.bin (40 MB) fills more than one pack: a restore to a directory reads ahead over several ranges
-$X sh -c "mkdir -p $DATA/$U/files/Docs/Sub && for i in 1 2 3 4 5; do head -c 300000 /dev/urandom > $DATA/$U/files/Docs/f\$i.bin; done; echo hello > $DATA/$U/files/Docs/Sub/note.txt; head -c 6000000 /dev/urandom > $DATA/$U/files/big.bin; head -c 40000000 /dev/urandom > $DATA/$U/files/large.bin"
+$X sh -c "mkdir -p $DATA/$U/files/Docs/Sub $DATA/$U/files/Docs/Empty/Nested $DATA/$U/files/Hollow && for i in 1 2 3 4 5; do head -c 300000 /dev/urandom > $DATA/$U/files/Docs/f\$i.bin; done; echo hello > $DATA/$U/files/Docs/Sub/note.txt; head -c 6000000 /dev/urandom > $DATA/$U/files/big.bin; head -c 40000000 /dev/urandom > $DATA/$U/files/large.bin"
 occ files:scan "$U" >/dev/null
 SUM_BEFORE=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
 
@@ -39,11 +39,17 @@ SNAP=$(occ backup:list ci | awk -F'|' '!f && / second /{gsub(/ /,"",$3); print $
 [ -n "$SNAP" ] || fail "snapshot not listed"
 
 # Damage, then restore in each mode.
-$X sh -c "cd $DATA/$U/files && rm Docs/Sub/note.txt && echo broken > Docs/f1.bin && echo extra > Docs/extra.txt"
+$X sh -c "cd $DATA/$U/files && rm Docs/Sub/note.txt && echo broken > Docs/f1.bin && echo extra > Docs/extra.txt && rm -r Docs/Empty && rmdir Hollow"
 occ files:scan "$U" >/dev/null
 occ backup:restore ci "$SNAP" "data/$U/files/Docs" -m new-folder >/dev/null || fail "restore new-folder"
 $X sh -c "ls $DATA/$U/files | grep -q '^Restored '" || fail "no Restored folder"
+$X sh -c "test -d $DATA/$U/files/Restored\ */Docs/Empty/Nested" || fail "the empty folder is not in the Restored folder"
+occ backup:browse ci "$SNAP" "data/$U/files" | grep Hollow >/dev/null || fail "an empty folder is not shown when browsing the snapshot"
+occ backup:restore ci "$SNAP" "data/$U/files/Hollow" -m merge >/dev/null || fail "restore of a folder that was empty"
+$X sh -c "test -d $DATA/$U/files/Hollow" || fail "the empty folder did not come back by itself"
 occ backup:restore ci "$SNAP" "data/$U/files/Docs" -m replace >/dev/null || fail "restore replace"
+$X sh -c "test -d $DATA/$U/files/Docs/Empty/Nested" || fail "replace did not bring back the empty folder, or moved it to the trash"
+ok "empty folders: restored in each mode, by themselves and shown when browsing"
 $X sh -c "rm -rf '$DATA/$U/files/Restored '*" ; occ files:scan "$U" >/dev/null
 SUM_AFTER=$($X sh -c "cd $DATA/$U/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
 [ "$SUM_BEFORE" = "$SUM_AFTER" ] || fail "files differ after restore"
@@ -63,6 +69,7 @@ occ backup:restore ci "$SNAP" "data/$U/files" -d /tmp/ngb-rt-dir >/dev/null || f
 SUM_DIR=$($X sh -c "cd /tmp/ngb-rt-dir/data/$U/files && find . -type f -print0 | sort -z | xargs -0 md5sum | md5sum")
 [ "$SUM_BEFORE" = "$SUM_DIR" ] || fail "the files restored to a directory differ"
 $X sh -c "ls -A /tmp/ngb-rt-dir | grep -q ngb-restore-spool" && fail "the read-ahead spool was left in the directory"
+$X sh -c "test -d /tmp/ngb-rt-dir/data/$U/files/Docs/Empty/Nested" || fail "the empty folder is not in the directory"
 $X sh -c "rm -rf /tmp/ngb-rt-dir"
 ok "restore of all the user's files to a directory: identical, no spool left"
 # not grep -q: it stops reading at the first match, and occ, still writing, then fails (pipefail)
@@ -90,6 +97,9 @@ done
 [ "$RS" = done ] || fail "the backup started through the API ended as '$RS' after $i ticks"
 occ backup:list ci | grep " page " >/dev/null || fail "the snapshot of the ticked backup is not listed"
 ok "without system cron: warning in backup:status and the API, backup finished by $i ticks of the open page"
+# The Bin reads in parts, as the page does (what is in the parts: CatalogTest)
+api "$API/targets/$TID/trash?offset=0" | php -r '$p = json_decode(stream_get_contents(STDIN), true); exit(is_int($p["total"] ?? null) && is_array($p["items"] ?? null) && count($p["items"]) <= $p["total"] ? 0 : 1);' || fail "the Bin API does not give a total and items"
+ok "the Bin API reads in parts, with a total"
 occ background:cron >/dev/null
 $X php -f /var/www/html/cron.php >/dev/null 2>&1 || true
 occ backup:status | grep "system cron" >/dev/null && fail "backup:status still warns after cron ran"