Phase 0: fixes from review (torn files, pending blobs, exact sequences, UTF-8, storage checks)

- detect files changing during reading or between resumable steps and
  re-read them (size/mtime/ctime/inode), spike F overwrites mid-run
- drop pending blob ids when a pack or index upload fails
- dump and restore PostgreSQL sequence state exactly, bound parameters,
  validated catalog names; spike C checks all sequences exactly
- keep invalid UTF-8 byte-exact (tagged base64)
- refuse large tables without a primary key
- StorageBackend: check write/flush/close results, validate base path
- user_migration import/export: root paths, intermediate folders,
  partial writes, read errors
- spikes: never touch an existing account, own fixtures, 512M note
This commit is contained in:
2026-10-01 18:58:51 +02:00
parent 06e55b08ea
commit 7b33347dbc
13 changed files with 274 additions and 70 deletions
+7
View File
@@ -55,6 +55,13 @@ final class BlobIndex {
$this->pending[$id] = true;
}
/** @param list<string> $ids */
public function dropPending(array $ids): void {
foreach ($ids as $id) {
unset($this->pending[$id]);
}
}
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
public function addPack(string $packId, array $entries): void {
$stream = fopen('php://memory', 'w+b');
+11 -3
View File
@@ -59,9 +59,17 @@ final class PackWriter {
fwrite($this->buffer, $header . pack('N', strlen($header)));
rewind($this->buffer);
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
$this->backend->put($path, $this->buffer); // closes the buffer
unset($this->buffer);
$this->index->addPack($packId, $this->entries);
try {
$this->backend->put($path, $this->buffer); // closes the buffer
unset($this->buffer);
$this->index->addPack($packId, $this->entries);
} catch (\Throwable $e) {
// The blobs of this pack are not durable: forget them, so a retry stores them again.
$this->index->dropPending(array_column($this->entries, 'id'));
unset($this->buffer);
$this->reset();
throw $e;
}
$this->reset();
}
+31 -6
View File
@@ -92,12 +92,7 @@ final class Repository {
$blobs = $prev['b'];
$stats['reused']++;
} else {
$blobs = [];
$fh = fopen($info->getPathname(), 'rb');
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
$blobs[] = $this->storeBlob($data, $packs, $stats);
}
fclose($fh);
$blobs = $this->readStable($info->getPathname(), $packs, $stats);
$stats['read']++;
}
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
@@ -244,6 +239,36 @@ final class Repository {
return $this->index->count();
}
/**
* Read a file into blobs and make sure it did not change while reading (size, mtime, ctime
* before and after). A file that keeps changing is an error rather than a torn backup.
*
* @return list<string>
*/
private function readStable(string $path, PackWriter $packs, array &$stats): array {
for ($attempt = 1; $attempt <= 3; $attempt++) {
$before = self::fileVersion($path);
$blobs = [];
$fh = fopen($path, 'rb');
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
$blobs[] = $this->storeBlob($data, $packs, $stats);
}
fclose($fh);
if (self::fileVersion($path) === $before) {
return $blobs;
}
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
}
throw new RepositoryException('File keeps changing while being read: ' . $path);
}
/** Cheap file identity: changes when the content may have changed. */
public static function fileVersion(string $path): string {
clearstatcache(true, $path);
$st = @stat($path);
return $st === false ? 'missing' : implode(':', [$st['size'], $st['mtime'], $st['ctime'], $st['ino']]);
}
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
$id = $this->keys->blobId($data);
if ($this->index->has($id)) {