Phase 0: repository on files_external storages without a mount (spike D)

ExternalStorageFactory builds storages with files_external's own backends and auth mechanisms from an unsaved StorageConfig (as its ConfigAdapter does), and StorageBackend adapts any IStorage to IBackend (writeStream, seekable range reads, temp name + rename except on S3). IBackend::put() now owns and always closes the stream, because files_external's S3 writeStream closes it. Spike D (NC34): S3 (SeaweedFS) and SFTP: 120 MiB backup ~6s, unchanged run uploads nothing, restore bit-identical, no files_external mount created, no temp files (max 12 KiB in /tmp). Works with files_external disabled when its classes are loaded. Backup peak memory 85-89 MiB on these targets (to investigate).
2026-10-01 18:33:15 +02:00
parent 915e534ab2
commit bc16a186d5
8 changed files with 339 additions and 5 deletions
@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+
+// SPDX-FileCopyrightText: 2026 Dillard Blom
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+namespace OCA\NgBackup\Backend;
+
+use OCA\Files_External\Lib\StorageConfig;
+use OCA\Files_External\Service\BackendService;
+use OCP\Files\Storage\IStorage;
+use OCP\IUser;
+
+/**
+ * Builds a storage object with files_external's own backends and auth mechanisms, the same
+ * way files_external's ConfigAdapter does it, but from a configuration that is never saved as
+ * a mount. The backup location therefore never appears in Files, for anyone.
+ *
+ * files_external classes are internal API: everything that touches them is in this class.
+ */
+final class ExternalStorageFactory {
+	public function __construct(
+		private BackendService $backends,
+	) {
+	}
+
+	/**
+	 * @return list<array{id:string, name:string, auth:list<string>, parameters:array<string, mixed>}>
+	 *         available backends, to build the settings form from
+	 */
+	public function describeBackends(): array {
+		$result = [];
+		foreach ($this->backends->getAvailableBackends() as $backend) {
+			$result[] = [
+				'id' => $backend->getIdentifier(),
+				'name' => $backend->getText(),
+				'auth' => array_keys($backend->getAuthSchemes()),
+				'parameters' => array_map(fn ($p) => $p->jsonSerialize(), $backend->getParameters()),
+			];
+		}
+		return $result;
+	}
+
+	/**
+	 * @param array<string, mixed> $options backend and auth options (e.g. host, bucket, key, secret)
+	 * @param IUser|null $admin user for auth mechanisms that need one (password/session-based ones are not supported)
+	 */
+	public function create(string $backendId, string $authId, array $options, ?IUser $admin = null): IStorage {
+		$backend = $this->backends->getBackend($backendId);
+		$auth = $this->backends->getAuthMechanism($authId);
+		if ($backend === null || $auth === null) {
+			throw new BackendException("Unknown storage backend or auth mechanism: $backendId / $authId");
+		}
+		if (!in_array($auth->getScheme(), array_keys($backend->getAuthSchemes()), true)) {
+			throw new BackendException("Auth mechanism $authId does not fit backend $backendId");
+		}
+
+		$config = new StorageConfig();
+		$config->setBackend($backend);
+		$config->setAuthMechanism($auth);
+		$config->setBackendOptions($options);
+		$config->setMountPoint('/ng_backup'); // never mounted, but some code expects a value
+
+		$auth->manipulateStorageConfig($config, $admin);
+		$backend->manipulateStorageConfig($config, $admin);
+
+		$class = $backend->getStorageClass();
+		$storage = new $class($config->getBackendOptions());
+		$storage = $backend->wrapStorage($storage);
+		return $auth->wrapStorage($storage);
+	}
+}
@@ -12,7 +12,11 @@ namespace OCA\NgBackup\Backend;
 * Implementations: a local directory (spike, tests), and adapters over files_external storages.
 */
 interface IBackend {
-	/** Store an object; the stream is read to EOF. Must not leave a partial object behind on failure. */
+	/**
+	 * Store an object; the stream is read to EOF. Takes ownership of the stream: it is always
+	 * closed afterwards (some storages, like files_external's S3, close it themselves).
+	 * Must not leave a partial object behind on failure.
+	 */
 	public function put(string $path, $stream): void;

 	/** @return resource readable stream of the whole object */
@@ -35,6 +35,9 @@ final class LocalBackend implements IBackend {
 			}
 		} finally {
 			fclose($out);
+			if (is_resource($stream)) {
+				fclose($stream);
+			}
 		}
 		if (!rename($tmp, $target)) {
 			@unlink($tmp);
@@ -0,0 +1,148 @@
+<?php
+
+declare(strict_types=1);
+
+// SPDX-FileCopyrightText: 2026 Dillard Blom
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+namespace OCA\NgBackup\Backend;
+
+use OCP\Files\Storage\IStorage;
+use OCP\Files\Storage\IWriteStreamStorage;
+
+/**
+ * IBackend on top of a Nextcloud storage object (from files_external or core), below a base
+ * directory on that storage.
+ */
+final class StorageBackend implements IBackend {
+	/** Object stores write whole objects atomically; other storages get a temp name + rename. */
+	private bool $atomicPut;
+
+	public function __construct(
+		private IStorage $storage,
+		private string $base = '',
+	) {
+		$this->base = trim($base, '/');
+		$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
+		if ($this->base !== '') {
+			$this->mkdirs($this->base);
+		}
+	}
+
+	public function put(string $path, $stream): void {
+		$target = $this->abs($path);
+		$this->mkdirs(dirname($target));
+		$writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4));
+		try {
+			if ($this->storage->instanceOfStorage(IWriteStreamStorage::class)) {
+				/** @var IWriteStreamStorage $s */
+				$s = $this->storage;
+				$s->writeStream($writeTo, $stream);
+			} else {
+				$out = $this->storage->fopen($writeTo, 'w');
+				if ($out === false || stream_copy_to_stream($stream, $out) === false) {
+					throw new BackendException('Write failed for ' . $path);
+				}
+				fclose($out);
+			}
+			if (is_resource($stream)) {
+				fclose($stream);
+			}
+			if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) {
+				throw new BackendException('Cannot finalise ' . $path);
+			}
+		} catch (\Throwable $e) {
+			if (is_resource($stream)) {
+				fclose($stream);
+			}
+			if ($writeTo !== $target) {
+				$this->storage->unlink($writeTo);
+			}
+			throw $e instanceof BackendException ? $e : new BackendException('Write failed for ' . $path . ': ' . $e->getMessage(), 0, $e);
+		}
+	}
+
+	public function get(string $path) {
+		$fh = $this->storage->fopen($this->abs($path), 'r');
+		if ($fh === false) {
+			throw new BackendException('Not found: ' . $path);
+		}
+		return $fh;
+	}
+
+	public function getRange(string $path, int $offset, int $length): string {
+		$fh = $this->get($path);
+		try {
+			if (fseek($fh, $offset) !== 0) {
+				throw new BackendException('Seek failed in ' . $path);
+			}
+			$data = '';
+			while (strlen($data) < $length && !feof($fh)) {
+				$chunk = fread($fh, $length - strlen($data));
+				if ($chunk === false) {
+					break;
+				}
+				$data .= $chunk;
+			}
+			if (strlen($data) !== $length) {
+				throw new BackendException('Short read in ' . $path);
+			}
+			return $data;
+		} finally {
+			fclose($fh);
+		}
+	}
+
+	public function exists(string $path): bool {
+		return $this->storage->file_exists($this->abs($path));
+	}
+
+	public function list(string $prefix): array {
+		$result = [];
+		$this->listInto($this->abs($prefix), $result);
+		sort($result);
+		$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
+		return array_map(fn ($p) => substr($p, $cut), $result);
+	}
+
+	public function delete(string $path): void {
+		$this->storage->unlink($this->abs($path));
+	}
+
+	private function listInto(string $dir, array &$result): void {
+		if (!$this->storage->is_dir($dir)) {
+			return;
+		}
+		$dh = $this->storage->opendir($dir);
+		if ($dh === false) {
+			return;
+		}
+		while (($name = readdir($dh)) !== false) {
+			if ($name === '.' || $name === '..' || str_contains($name, '.part-')) {
+				continue;
+			}
+			$child = $dir . '/' . $name;
+			if ($this->storage->is_dir($child)) {
+				$this->listInto($child, $result);
+			} else {
+				$result[] = $child;
+			}
+		}
+		closedir($dh);
+	}
+
+	private function mkdirs(string $dir): void {
+		if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) {
+			return;
+		}
+		$this->mkdirs(dirname($dir) === '.' ? '' : dirname($dir));
+		$this->storage->mkdir($dir);
+	}
+
+	private function abs(string $path): string {
+		if (str_contains($path, '..')) {
+			throw new BackendException('Invalid path: ' . $path);
+		}
+		return ltrim(($this->base === '' ? '' : $this->base . '/') . ltrim($path, '/'), '/');
+	}
+}