Phase 0: fixes from review (torn files, pending blobs, exact sequences, UTF-8, storage checks)
- detect files changing during reading or between resumable steps and re-read them (size/mtime/ctime/inode), spike F overwrites mid-run - drop pending blob ids when a pack or index upload fails - dump and restore PostgreSQL sequence state exactly, bound parameters, validated catalog names; spike C checks all sequences exactly - keep invalid UTF-8 byte-exact (tagged base64) - refuse large tables without a primary key - StorageBackend: check write/flush/close results, validate base path - user_migration import/export: root paths, intermediate folders, partial writes, read errors - spikes: never touch an existing account, own fixtures, 512M note
This commit is contained in:
@@ -55,6 +55,13 @@ final class BlobIndex {
|
||||
$this->pending[$id] = true;
|
||||
}
|
||||
|
||||
/** @param list<string> $ids */
|
||||
public function dropPending(array $ids): void {
|
||||
foreach ($ids as $id) {
|
||||
unset($this->pending[$id]);
|
||||
}
|
||||
}
|
||||
|
||||
/** @param list<array{id:string, offset:int, length:int, raw:int, flags:int}> $entries */
|
||||
public function addPack(string $packId, array $entries): void {
|
||||
$stream = fopen('php://memory', 'w+b');
|
||||
|
||||
@@ -59,9 +59,17 @@ final class PackWriter {
|
||||
fwrite($this->buffer, $header . pack('N', strlen($header)));
|
||||
rewind($this->buffer);
|
||||
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
|
||||
$this->backend->put($path, $this->buffer); // closes the buffer
|
||||
unset($this->buffer);
|
||||
$this->index->addPack($packId, $this->entries);
|
||||
try {
|
||||
$this->backend->put($path, $this->buffer); // closes the buffer
|
||||
unset($this->buffer);
|
||||
$this->index->addPack($packId, $this->entries);
|
||||
} catch (\Throwable $e) {
|
||||
// The blobs of this pack are not durable: forget them, so a retry stores them again.
|
||||
$this->index->dropPending(array_column($this->entries, 'id'));
|
||||
unset($this->buffer);
|
||||
$this->reset();
|
||||
throw $e;
|
||||
}
|
||||
$this->reset();
|
||||
}
|
||||
|
||||
|
||||
@@ -92,12 +92,7 @@ final class Repository {
|
||||
$blobs = $prev['b'];
|
||||
$stats['reused']++;
|
||||
} else {
|
||||
$blobs = [];
|
||||
$fh = fopen($info->getPathname(), 'rb');
|
||||
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
|
||||
$blobs[] = $this->storeBlob($data, $packs, $stats);
|
||||
}
|
||||
fclose($fh);
|
||||
$blobs = $this->readStable($info->getPathname(), $packs, $stats);
|
||||
$stats['read']++;
|
||||
}
|
||||
fwrite($tree, json_encode(['p' => $relative, 's' => $info->getSize(), 'm' => $info->getMTime(), 'b' => $blobs], JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n");
|
||||
@@ -244,6 +239,36 @@ final class Repository {
|
||||
return $this->index->count();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a file into blobs and make sure it did not change while reading (size, mtime, ctime
|
||||
* before and after). A file that keeps changing is an error rather than a torn backup.
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
private function readStable(string $path, PackWriter $packs, array &$stats): array {
|
||||
for ($attempt = 1; $attempt <= 3; $attempt++) {
|
||||
$before = self::fileVersion($path);
|
||||
$blobs = [];
|
||||
$fh = fopen($path, 'rb');
|
||||
while (($data = self::readFull($fh, self::BLOB_SIZE)) !== '') {
|
||||
$blobs[] = $this->storeBlob($data, $packs, $stats);
|
||||
}
|
||||
fclose($fh);
|
||||
if (self::fileVersion($path) === $before) {
|
||||
return $blobs;
|
||||
}
|
||||
$stats['retried'] = ($stats['retried'] ?? 0) + 1;
|
||||
}
|
||||
throw new RepositoryException('File keeps changing while being read: ' . $path);
|
||||
}
|
||||
|
||||
/** Cheap file identity: changes when the content may have changed. */
|
||||
public static function fileVersion(string $path): string {
|
||||
clearstatcache(true, $path);
|
||||
$st = @stat($path);
|
||||
return $st === false ? 'missing' : implode(':', [$st['size'], $st['mtime'], $st['ctime'], $st['ino']]);
|
||||
}
|
||||
|
||||
private function storeBlob(string $data, PackWriter $packs, array &$stats): string {
|
||||
$id = $this->keys->blobId($data);
|
||||
if ($this->index->has($id)) {
|
||||
|
||||
Reference in New Issue
Block a user