Phase 0: fixes from review (torn files, pending blobs, exact sequences, UTF-8, storage checks)

- detect files changing during reading or between resumable steps and
  re-read them (size/mtime/ctime/inode), spike F overwrites mid-run
- drop pending blob ids when a pack or index upload fails
- dump and restore PostgreSQL sequence state exactly, bound parameters,
  validated catalog names; spike C checks all sequences exactly
- keep invalid UTF-8 byte-exact (tagged base64)
- refuse large tables without a primary key
- StorageBackend: check write/flush/close results, validate base path
- user_migration import/export: root paths, intermediate folders,
  partial writes, read errors
- spikes: never touch an existing account, own fixtures, 512M note
This commit is contained in:
2026-10-01 18:58:51 +02:00
parent 06e55b08ea
commit 7b33347dbc
13 changed files with 274 additions and 70 deletions
+31 -6
View File
@@ -92,12 +92,7 @@ final class Repository {
$blobs = $prev['b'];
$stats['reused']++;
} else {
$blobs = [];
$fh = fopen($info->getPathname(), 'rb');
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
$blobs[] = $this->storeBlob($data, $packs, $stats);
}
fclose($fh);
$blobs = $this->readStable($info->getPathname(), $packs, $stats);
$stats['read']++;
}
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
@@ -244,6 +239,36 @@ final class Repository {
return $this->index->count();
}
/**
* Read a file into blobs and make sure it did not change while reading (size, mtime, ctime
* before and after). A file that keeps changing is an error rather than a torn backup.
*
* @return list<string>
*/
private function readStable(string $path, PackWriter $packs, array &$stats): array {
for ($attempt = 1; $attempt <= 3; $attempt++) {
$before = self::fileVersion($path);
$blobs = [];
$fh = fopen($path, 'rb');
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
$blobs[] = $this->storeBlob($data, $packs, $stats);
}
fclose($fh);
if (self::fileVersion($path) === $before) {
return $blobs;
}
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
}
throw new RepositoryException('File keeps changing while being read: ' . $path);
}
/** Cheap file identity: changes when the content may have changed. */
public static function fileVersion(string $path): string {
clearstatcache(true, $path);
$st = @stat($path);
return $st === false ? 'missing' : implode(':', [$st['size'], $st['mtime'], $st['ctime'], $st['ino']]);
}
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
$id = $this->keys->blobId($data);
if ($this->index->has($id)) {