diff --git a/lib/Backend/ExternalStorageFactory.php b/lib/Backend/ExternalStorageFactory.php new file mode 100644 index 0000000..095b790 --- /dev/null +++ b/lib/Backend/ExternalStorageFactory.php @@ -0,0 +1,73 @@ +, parameters:array}> + * available backends, to build the settings form from + */ + public function describeBackends(): array { + $result = []; + foreach ($this->backends->getAvailableBackends() as $backend) { + $result[] = [ + 'id' => $backend->getIdentifier(), + 'name' => $backend->getText(), + 'auth' => array_keys($backend->getAuthSchemes()), + 'parameters' => array_map(fn ($p) => $p->jsonSerialize(), $backend->getParameters()), + ]; + } + return $result; + } + + /** + * @param array $options backend and auth options (e.g. host, bucket, key, secret) + * @param IUser|null $admin user for auth mechanisms that need one (password/session-based ones are not supported) + */ + public function create(string $backendId, string $authId, array $options, ?IUser $admin = null): IStorage { + $backend = $this->backends->getBackend($backendId); + $auth = $this->backends->getAuthMechanism($authId); + if ($backend === null || $auth === null) { + throw new BackendException("Unknown storage backend or auth mechanism: $backendId / $authId"); + } + if (!in_array($auth->getScheme(), array_keys($backend->getAuthSchemes()), true)) { + throw new BackendException("Auth mechanism $authId does not fit backend $backendId"); + } + + $config = new StorageConfig(); + $config->setBackend($backend); + $config->setAuthMechanism($auth); + $config->setBackendOptions($options); + $config->setMountPoint('/ng_backup'); // never mounted, but some code expects a value + + $auth->manipulateStorageConfig($config, $admin); + $backend->manipulateStorageConfig($config, $admin); + + $class = $backend->getStorageClass(); + $storage = new $class($config->getBackendOptions()); + $storage = $backend->wrapStorage($storage); + return $auth->wrapStorage($storage); + } +} diff --git a/lib/Backend/IBackend.php b/lib/Backend/IBackend.php index 08b6459..f52eef4 100644 --- a/lib/Backend/IBackend.php +++ b/lib/Backend/IBackend.php @@ -12,7 +12,11 @@ namespace OCA\NgBackup\Backend; * Implementations: a local directory (spike, tests), and adapters over files_external storages. */ interface IBackend { - /** Store an object; the stream is read to EOF. Must not leave a partial object behind on failure. */ + /** + * Store an object; the stream is read to EOF. Takes ownership of the stream: it is always + * closed afterwards (some storages, like files_external's S3, close it themselves). + * Must not leave a partial object behind on failure. + */ public function put(string $path, $stream): void; /** @return resource readable stream of the whole object */ diff --git a/lib/Backend/LocalBackend.php b/lib/Backend/LocalBackend.php index 17960bb..e759d85 100644 --- a/lib/Backend/LocalBackend.php +++ b/lib/Backend/LocalBackend.php @@ -35,6 +35,9 @@ final class LocalBackend implements IBackend { } } finally { fclose($out); + if (is_resource($stream)) { + fclose($stream); + } } if (!rename($tmp, $target)) { @unlink($tmp); diff --git a/lib/Backend/StorageBackend.php b/lib/Backend/StorageBackend.php new file mode 100644 index 0000000..4c5deae --- /dev/null +++ b/lib/Backend/StorageBackend.php @@ -0,0 +1,148 @@ +base = trim($base, '/'); + $this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class); + if ($this->base !== '') { + $this->mkdirs($this->base); + } + } + + public function put(string $path, $stream): void { + $target = $this->abs($path); + $this->mkdirs(dirname($target)); + $writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4)); + try { + if ($this->storage->instanceOfStorage(IWriteStreamStorage::class)) { + /** @var IWriteStreamStorage $s */ + $s = $this->storage; + $s->writeStream($writeTo, $stream); + } else { + $out = $this->storage->fopen($writeTo, 'w'); + if ($out === false || stream_copy_to_stream($stream, $out) === false) { + throw new BackendException('Write failed for ' . $path); + } + fclose($out); + } + if (is_resource($stream)) { + fclose($stream); + } + if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) { + throw new BackendException('Cannot finalise ' . $path); + } + } catch (\Throwable $e) { + if (is_resource($stream)) { + fclose($stream); + } + if ($writeTo !== $target) { + $this->storage->unlink($writeTo); + } + throw $e instanceof BackendException ? $e : new BackendException('Write failed for ' . $path . ': ' . $e->getMessage(), 0, $e); + } + } + + public function get(string $path) { + $fh = $this->storage->fopen($this->abs($path), 'r'); + if ($fh === false) { + throw new BackendException('Not found: ' . $path); + } + return $fh; + } + + public function getRange(string $path, int $offset, int $length): string { + $fh = $this->get($path); + try { + if (fseek($fh, $offset) !== 0) { + throw new BackendException('Seek failed in ' . $path); + } + $data = ''; + while (strlen($data) < $length && !feof($fh)) { + $chunk = fread($fh, $length - strlen($data)); + if ($chunk === false) { + break; + } + $data .= $chunk; + } + if (strlen($data) !== $length) { + throw new BackendException('Short read in ' . $path); + } + return $data; + } finally { + fclose($fh); + } + } + + public function exists(string $path): bool { + return $this->storage->file_exists($this->abs($path)); + } + + public function list(string $prefix): array { + $result = []; + $this->listInto($this->abs($prefix), $result); + sort($result); + $cut = $this->base === '' ? 0 : strlen($this->base) + 1; + return array_map(fn ($p) => substr($p, $cut), $result); + } + + public function delete(string $path): void { + $this->storage->unlink($this->abs($path)); + } + + private function listInto(string $dir, array &$result): void { + if (!$this->storage->is_dir($dir)) { + return; + } + $dh = $this->storage->opendir($dir); + if ($dh === false) { + return; + } + while (($name = readdir($dh)) !== false) { + if ($name === '.' || $name === '..' || str_contains($name, '.part-')) { + continue; + } + $child = $dir . '/' . $name; + if ($this->storage->is_dir($child)) { + $this->listInto($child, $result); + } else { + $result[] = $child; + } + } + closedir($dh); + } + + private function mkdirs(string $dir): void { + if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) { + return; + } + $this->mkdirs(dirname($dir) === '.' ? '' : dirname($dir)); + $this->storage->mkdir($dir); + } + + private function abs(string $path): string { + if (str_contains($path, '..')) { + throw new BackendException('Invalid path: ' . $path); + } + return ltrim(($this->base === '' ? '' : $this->base . '/') . ltrim($path, '/'), '/'); + } +} diff --git a/lib/Repository/BlobIndex.php b/lib/Repository/BlobIndex.php index bccacf3..d67697e 100644 --- a/lib/Repository/BlobIndex.php +++ b/lib/Repository/BlobIndex.php @@ -61,7 +61,6 @@ final class BlobIndex { fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId)); rewind($stream); $this->backend->put('index/' . $packId, $stream); - fclose($stream); foreach ($entries as $e) { $this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']]; unset($this->pending[$e['id']]); diff --git a/lib/Repository/PackWriter.php b/lib/Repository/PackWriter.php index 08f142e..882b9da 100644 --- a/lib/Repository/PackWriter.php +++ b/lib/Repository/PackWriter.php @@ -59,7 +59,8 @@ final class PackWriter { fwrite($this->buffer, $header . pack('N', strlen($header))); rewind($this->buffer); $path = 'packs/' . substr($packId, 0, 2) . '/' . $packId; - $this->backend->put($path, $this->buffer); + $this->backend->put($path, $this->buffer); // closes the buffer + unset($this->buffer); $this->index->addPack($packId, $this->entries); $this->reset(); } diff --git a/lib/Repository/Repository.php b/lib/Repository/Repository.php index 6d51d3d..ecab14d 100644 --- a/lib/Repository/Repository.php +++ b/lib/Repository/Repository.php @@ -111,7 +111,6 @@ final class Repository { rewind($encrypted); $this->backend->put('snapshots/' . $snapshotId, $encrypted); fclose($tree); - fclose($encrypted); return ['snapshot' => $snapshotId] + $stats; } @@ -272,7 +271,6 @@ final class Repository { fwrite($s, $data); rewind($s); $backend->put($path, $s); - fclose($s); } private static function readFull($fh, int $length): string { diff --git a/spikes/04-external-storage.php b/spikes/04-external-storage.php new file mode 100644 index 0000000..ce64f6d --- /dev/null +++ b/spikes/04-external-storage.php @@ -0,0 +1,108 @@ +isFile()) { + $out[substr($f->getPathname(), strlen($root) + 1)] = hash_file('sha256', $f->getPathname()); + } + } + ksort($out); + return $out; +}; + +// Source tree: 100 small files + one 120 MiB file. +$src = '/tmp/ngb-d-src'; +exec('rm -rf /tmp/ngb-d-src /tmp/ngb-d-restore-*'); +mkdir("$src/docs", 0700, true); +for ($i = 0; $i < 100; $i++) { + file_put_contents("$src/docs/note-$i.txt", str_repeat("notitie $i ", 2000)); +} +$fh = fopen("$src/big.bin", 'wb'); +for ($i = 0; $i < 120; $i++) { + fwrite($fh, random_bytes(1048576)); +} +fclose($fh); +$state = $hashTree($src); + +$factory = new ExternalStorageFactory(\OCP\Server::get(BackendService::class)); +$names = array_column($factory->describeBackends(), 'id'); +printf("files_external backends available: %s\n", implode(', ', $names)); + +$targets = [ + 's3' => ['amazons3', 'amazons3::accesskey', [ + 'bucket' => 'ngbackup', 'hostname' => 's3', 'port' => '8333', 'region' => 'us-east-1', + 'use_ssl' => false, 'use_path_style' => true, 'autocreate' => true, + 'key' => 'ngbtestkey', 'secret' => 'ngbtestsecret123', + ]], + 'sftp' => ['sftp', 'password::password', [ + 'host' => 'sftp', 'root' => '/repo', 'user' => 'ngb', 'password' => 'ngbtestpass', + ]], +]; + +$mountsBefore = (int)\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder() + ->select($q = \OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()->func()->count('*')) + ->from('external_mounts')->executeQuery()->fetchOne(); + +foreach ($targets as $name => [$backendId, $authId, $options]) { + if ($which !== 'both' && $which !== $name) { + continue; + } + echo "--- $name\n"; + $storage = $factory->create($backendId, $authId, $options); + $backend = new StorageBackend($storage, 'spike-' . bin2hex(random_bytes(3))); + + memory_reset_peak_usage(); + $t = microtime(true); + $repo = Repository::init($backend, 'spike passphrase'); + $r1 = $repo->backupDirectory($src, null, $name); + printf("backup: %.1fs, peak %.1f MiB, uploaded %.1f MiB\n", microtime(true) - $t, memory_get_peak_usage() / 1048576, $r1['uploaded'] / 1048576); + + $t = microtime(true); + $r2 = $repo->backupDirectory($src, $r1['snapshot'], $name); + printf("unchanged run: %.2fs, uploaded %d bytes\n", microtime(true) - $t, $r2['uploaded']); + + memory_reset_peak_usage(); + $t = microtime(true); + $repo = Repository::open($backend, 'spike passphrase'); + $repo->restore($r1['snapshot'], "/tmp/ngb-d-restore-$name"); + printf("restore: %.1fs, peak %.1f MiB\n", microtime(true) - $t, memory_get_peak_usage() / 1048576); + $check("$name: restored tree identical", $hashTree("/tmp/ngb-d-restore-$name") === $state); + $check("$name: unchanged run uploads nothing", $r2['uploaded'] === 0); + $check("$name: peak memory < 64 MiB", memory_get_peak_usage() < 64 * 1048576); + $check("$name: objects on target: config, packs, index, snapshots", count($backend->list('snapshots')) === 2 && $backend->exists('config')); +} + +$mountsAfter = (int)\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder() + ->select(\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()->func()->count('*')) + ->from('external_mounts')->executeQuery()->fetchOne(); +$check("no files_external mount created ($mountsBefore -> $mountsAfter)", $mountsBefore === $mountsAfter); + +exec('rm -rf /tmp/ngb-d-src /tmp/ngb-d-restore-*'); +echo $ok ? "ALL PASS\n" : "FAILURES\n"; +exit($ok ? 0 : 1);