Phase 0: repository on files_external storages without a mount (spike D)

ExternalStorageFactory builds storages with files_external's own
backends and auth mechanisms from an unsaved StorageConfig (as its
ConfigAdapter does), and StorageBackend adapts any IStorage to IBackend
(writeStream, seekable range reads, temp name + rename except on S3).
IBackend::put() now owns and always closes the stream, because
files_external's S3 writeStream closes it.

Spike D (NC34): S3 (SeaweedFS) and SFTP: 120 MiB backup ~6s, unchanged
run uploads nothing, restore bit-identical, no files_external mount
created, no temp files (max 12 KiB in /tmp). Works with files_external
disabled when its classes are loaded. Backup peak memory 85-89 MiB on
these targets (to investigate).

lib/Backend/ExternalStorageFactory.php

@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+
+// SPDX-FileCopyrightText: 2026 Dillard Blom
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+namespace OCA\NgBackup\Backend;
+
+use OCA\Files_External\Lib\StorageConfig;
+use OCA\Files_External\Service\BackendService;
+use OCP\Files\Storage\IStorage;
+use OCP\IUser;
+
+/**
+ * Builds a storage object with files_external's own backends and auth mechanisms, the same
+ * way files_external's ConfigAdapter does it, but from a configuration that is never saved as
+ * a mount. The backup location therefore never appears in Files, for anyone.
+ *
+ * files_external classes are internal API: everything that touches them is in this class.
+ */
+final class ExternalStorageFactory {
+	public function __construct(
+		private BackendService $backends,
+	) {
+	}
+
+	/**
+	 * @return list<array{id:string, name:string, auth:list<string>, parameters:array<string, mixed>}>
+	 *         available backends, to build the settings form from
+	 */
+	public function describeBackends(): array {
+		$result = [];
+		foreach ($this->backends->getAvailableBackends() as $backend) {
+			$result[] = [
+				'id' => $backend->getIdentifier(),
+				'name' => $backend->getText(),
+				'auth' => array_keys($backend->getAuthSchemes()),
+				'parameters' => array_map(fn ($p) => $p->jsonSerialize(), $backend->getParameters()),
+			];
+		}
+		return $result;
+	}
+
+	/**
+	 * @param array<string, mixed> $options backend and auth options (e.g. host, bucket, key, secret)
+	 * @param IUser|null $admin user for auth mechanisms that need one (password/session-based ones are not supported)
+	 */
+	public function create(string $backendId, string $authId, array $options, ?IUser $admin = null): IStorage {
+		$backend = $this->backends->getBackend($backendId);
+		$auth = $this->backends->getAuthMechanism($authId);
+		if ($backend === null || $auth === null) {
+			throw new BackendException("Unknown storage backend or auth mechanism: $backendId / $authId");
+		}
+		if (!in_array($auth->getScheme(), array_keys($backend->getAuthSchemes()), true)) {
+			throw new BackendException("Auth mechanism $authId does not fit backend $backendId");
+		}
+
+		$config = new StorageConfig();
+		$config->setBackend($backend);
+		$config->setAuthMechanism($auth);
+		$config->setBackendOptions($options);
+		$config->setMountPoint('/ng_backup'); // never mounted, but some code expects a value
+
+		$auth->manipulateStorageConfig($config, $admin);
+		$backend->manipulateStorageConfig($config, $admin);
+
+		$class = $backend->getStorageClass();
+		$storage = new $class($config->getBackendOptions());
+		$storage = $backend->wrapStorage($storage);
+		return $auth->wrapStorage($storage);
+	}
+}

lib/Backend/IBackend.php

@@ -12,7 +12,11 @@ namespace OCA\NgBackup\Backend;
  * Implementations: a local directory (spike, tests), and adapters over files_external storages.
  */
 interface IBackend {
-	/** Store an object; the stream is read to EOF. Must not leave a partial object behind on failure. */
+	/**
+	 * Store an object; the stream is read to EOF. Takes ownership of the stream: it is always
+	 * closed afterwards (some storages, like files_external's S3, close it themselves).
+	 * Must not leave a partial object behind on failure.
+	 */
 	public function put(string $path, $stream): void;
 
 	/** @return resource readable stream of the whole object */

lib/Backend/LocalBackend.php

@@ -35,6 +35,9 @@ final class LocalBackend implements IBackend {
 			}
 		} finally {
 			fclose($out);
+			if (is_resource($stream)) {
+				fclose($stream);
+			}
 		}
 		if (!rename($tmp, $target)) {
 			@unlink($tmp);

lib/Backend/StorageBackend.php

@@ -0,0 +1,148 @@
+<?php
+
+declare(strict_types=1);
+
+// SPDX-FileCopyrightText: 2026 Dillard Blom
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+namespace OCA\NgBackup\Backend;
+
+use OCP\Files\Storage\IStorage;
+use OCP\Files\Storage\IWriteStreamStorage;
+
+/**
+ * IBackend on top of a Nextcloud storage object (from files_external or core), below a base
+ * directory on that storage.
+ */
+final class StorageBackend implements IBackend {
+	/** Object stores write whole objects atomically; other storages get a temp name + rename. */
+	private bool $atomicPut;
+
+	public function __construct(
+		private IStorage $storage,
+		private string $base = '',
+	) {
+		$this->base = trim($base, '/');
+		$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
+		if ($this->base !== '') {
+			$this->mkdirs($this->base);
+		}
+	}
+
+	public function put(string $path, $stream): void {
+		$target = $this->abs($path);
+		$this->mkdirs(dirname($target));
+		$writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4));
+		try {
+			if ($this->storage->instanceOfStorage(IWriteStreamStorage::class)) {
+				/** @var IWriteStreamStorage $s */
+				$s = $this->storage;
+				$s->writeStream($writeTo, $stream);
+			} else {
+				$out = $this->storage->fopen($writeTo, 'w');
+				if ($out === false || stream_copy_to_stream($stream, $out) === false) {
+					throw new BackendException('Write failed for ' . $path);
+				}
+				fclose($out);
+			}
+			if (is_resource($stream)) {
+				fclose($stream);
+			}
+			if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) {
+				throw new BackendException('Cannot finalise ' . $path);
+			}
+		} catch (\Throwable $e) {
+			if (is_resource($stream)) {
+				fclose($stream);
+			}
+			if ($writeTo !== $target) {
+				$this->storage->unlink($writeTo);
+			}
+			throw $e instanceof BackendException ? $e : new BackendException('Write failed for ' . $path . ': ' . $e->getMessage(), 0, $e);
+		}
+	}
+
+	public function get(string $path) {
+		$fh = $this->storage->fopen($this->abs($path), 'r');
+		if ($fh === false) {
+			throw new BackendException('Not found: ' . $path);
+		}
+		return $fh;
+	}
+
+	public function getRange(string $path, int $offset, int $length): string {
+		$fh = $this->get($path);
+		try {
+			if (fseek($fh, $offset) !== 0) {
+				throw new BackendException('Seek failed in ' . $path);
+			}
+			$data = '';
+			while (strlen($data) < $length && !feof($fh)) {
+				$chunk = fread($fh, $length - strlen($data));
+				if ($chunk === false) {
+					break;
+				}
+				$data .= $chunk;
+			}
+			if (strlen($data) !== $length) {
+				throw new BackendException('Short read in ' . $path);
+			}
+			return $data;
+		} finally {
+			fclose($fh);
+		}
+	}
+
+	public function exists(string $path): bool {
+		return $this->storage->file_exists($this->abs($path));
+	}
+
+	public function list(string $prefix): array {
+		$result = [];
+		$this->listInto($this->abs($prefix), $result);
+		sort($result);
+		$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
+		return array_map(fn ($p) => substr($p, $cut), $result);
+	}
+
+	public function delete(string $path): void {
+		$this->storage->unlink($this->abs($path));
+	}
+
+	private function listInto(string $dir, array &$result): void {
+		if (!$this->storage->is_dir($dir)) {
+			return;
+		}
+		$dh = $this->storage->opendir($dir);
+		if ($dh === false) {
+			return;
+		}
+		while (($name = readdir($dh)) !== false) {
+			if ($name === '.' || $name === '..' || str_contains($name, '.part-')) {
+				continue;
+			}
+			$child = $dir . '/' . $name;
+			if ($this->storage->is_dir($child)) {
+				$this->listInto($child, $result);
+			} else {
+				$result[] = $child;
+			}
+		}
+		closedir($dh);
+	}
+
+	private function mkdirs(string $dir): void {
+		if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) {
+			return;
+		}
+		$this->mkdirs(dirname($dir) === '.' ? '' : dirname($dir));
+		$this->storage->mkdir($dir);
+	}
+
+	private function abs(string $path): string {
+		if (str_contains($path, '..')) {
+			throw new BackendException('Invalid path: ' . $path);
+		}
+		return ltrim(($this->base === '' ? '' : $this->base . '/') . ltrim($path, '/'), '/');
+	}
+}

lib/Repository/BlobIndex.php

@@ -61,7 +61,6 @@ final class BlobIndex {
 		fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
 		rewind($stream);
 		$this->backend->put('index/' . $packId, $stream);
-		fclose($stream);
 		foreach ($entries as $e) {
 			$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
 			unset($this->pending[$e['id']]);

lib/Repository/PackWriter.php

@@ -59,7 +59,8 @@ final class PackWriter {
 		fwrite($this->buffer, $header . pack('N', strlen($header)));
 		rewind($this->buffer);
 		$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
-		$this->backend->put($path, $this->buffer);
+		$this->backend->put($path, $this->buffer); // closes the buffer
+		unset($this->buffer);
 		$this->index->addPack($packId, $this->entries);
 		$this->reset();
 	}

lib/Repository/Repository.php

@@ -111,7 +111,6 @@ final class Repository {
 		rewind($encrypted);
 		$this->backend->put('snapshots/' . $snapshotId, $encrypted);
 		fclose($tree);
-		fclose($encrypted);
 
 		return ['snapshot' => $snapshotId] + $stats;
 	}
@@ -272,7 +271,6 @@ final class Repository {
 		fwrite($s, $data);
 		rewind($s);
 		$backend->put($path, $s);
-		fclose($s);
 	}
 
 	private static function readFull($fh, int $length): string {

spikes/04-external-storage.php

@@ -0,0 +1,108 @@
+<?php
+
+declare(strict_types=1);
+
+// Spike D: repository on S3 and SFTP through files_external classes, without any mount.
+// Run inside the Nextcloud container as the web user:
+//   php -d memory_limit=128M /var/www/html/custom_apps/ng_backup/spikes/04-external-storage.php [s3|sftp|both]
+
+require '/var/www/html/lib/base.php';
+set_exception_handler(null);
+restore_error_handler();
+require __DIR__ . '/bootstrap.php';
+
+use OCA\Files_External\Service\BackendService;
+use OCA\NgBackup\Backend\ExternalStorageFactory;
+use OCA\NgBackup\Backend\StorageBackend;
+use OCA\NgBackup\Repository\Repository;
+
+\OC_App::loadApp('files_external');
+
+$which = $argv[1] ?? 'both';
+$ok = true;
+$check = static function (string $name, bool $pass) use (&$ok): void {
+	printf("%-62s %s\n", $name, $pass ? 'PASS' : 'FAIL');
+	$ok = $ok && $pass;
+};
+$hashTree = static function (string $root): array {
+	$out = [];
+	$it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root, FilesystemIterator::SKIP_DOTS));
+	foreach ($it as $f) {
+		if ($f->isFile()) {
+			$out[substr($f->getPathname(), strlen($root) + 1)] = hash_file('sha256', $f->getPathname());
+		}
+	}
+	ksort($out);
+	return $out;
+};
+
+// Source tree: 100 small files + one 120 MiB file.
+$src = '/tmp/ngb-d-src';
+exec('rm -rf /tmp/ngb-d-src /tmp/ngb-d-restore-*');
+mkdir("$src/docs", 0700, true);
+for ($i = 0; $i < 100; $i++) {
+	file_put_contents("$src/docs/note-$i.txt", str_repeat("notitie $i ", 2000));
+}
+$fh = fopen("$src/big.bin", 'wb');
+for ($i = 0; $i < 120; $i++) {
+	fwrite($fh, random_bytes(1048576));
+}
+fclose($fh);
+$state = $hashTree($src);
+
+$factory = new ExternalStorageFactory(\OCP\Server::get(BackendService::class));
+$names = array_column($factory->describeBackends(), 'id');
+printf("files_external backends available: %s\n", implode(', ', $names));
+
+$targets = [
+	's3' => ['amazons3', 'amazons3::accesskey', [
+		'bucket' => 'ngbackup', 'hostname' => 's3', 'port' => '8333', 'region' => 'us-east-1',
+		'use_ssl' => false, 'use_path_style' => true, 'autocreate' => true,
+		'key' => 'ngbtestkey', 'secret' => 'ngbtestsecret123',
+	]],
+	'sftp' => ['sftp', 'password::password', [
+		'host' => 'sftp', 'root' => '/repo', 'user' => 'ngb', 'password' => 'ngbtestpass',
+	]],
+];
+
+$mountsBefore = (int)\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()
+	->select($q = \OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()->func()->count('*'))
+	->from('external_mounts')->executeQuery()->fetchOne();
+
+foreach ($targets as $name => [$backendId, $authId, $options]) {
+	if ($which !== 'both' && $which !== $name) {
+		continue;
+	}
+	echo "--- $name\n";
+	$storage = $factory->create($backendId, $authId, $options);
+	$backend = new StorageBackend($storage, 'spike-' . bin2hex(random_bytes(3)));
+
+	memory_reset_peak_usage();
+	$t = microtime(true);
+	$repo = Repository::init($backend, 'spike passphrase');
+	$r1 = $repo->backupDirectory($src, null, $name);
+	printf("backup: %.1fs, peak %.1f MiB, uploaded %.1f MiB\n", microtime(true) - $t, memory_get_peak_usage() / 1048576, $r1['uploaded'] / 1048576);
+
+	$t = microtime(true);
+	$r2 = $repo->backupDirectory($src, $r1['snapshot'], $name);
+	printf("unchanged run: %.2fs, uploaded %d bytes\n", microtime(true) - $t, $r2['uploaded']);
+
+	memory_reset_peak_usage();
+	$t = microtime(true);
+	$repo = Repository::open($backend, 'spike passphrase');
+	$repo->restore($r1['snapshot'], "/tmp/ngb-d-restore-$name");
+	printf("restore: %.1fs, peak %.1f MiB\n", microtime(true) - $t, memory_get_peak_usage() / 1048576);
+	$check("$name: restored tree identical", $hashTree("/tmp/ngb-d-restore-$name") === $state);
+	$check("$name: unchanged run uploads nothing", $r2['uploaded'] === 0);
+	$check("$name: peak memory < 64 MiB", memory_get_peak_usage() < 64 * 1048576);
+	$check("$name: objects on target: config, packs, index, snapshots", count($backend->list('snapshots')) === 2 && $backend->exists('config'));
+}
+
+$mountsAfter = (int)\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()
+	->select(\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()->func()->count('*'))
+	->from('external_mounts')->executeQuery()->fetchOne();
+$check("no files_external mount created ($mountsBefore -> $mountsAfter)", $mountsBefore === $mountsAfter);
+
+exec('rm -rf /tmp/ngb-d-src /tmp/ngb-d-restore-*');
+echo $ok ? "ALL PASS\n" : "FAILURES\n";
+exit($ok ? 0 : 1);