Ensembia/nextcloud-ng_backup
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 17 Wiki Activity

Phase 0: repository on files_external storages without a mount (spike D)

Browse Source 
ExternalStorageFactory builds storages with files_external's own
backends and auth mechanisms from an unsaved StorageConfig (as its
ConfigAdapter does), and StorageBackend adapts any IStorage to IBackend
(writeStream, seekable range reads, temp name + rename except on S3).
IBackend::put() now owns and always closes the stream, because
files_external's S3 writeStream closes it.

Spike D (NC34): S3 (SeaweedFS) and SFTP: 120 MiB backup ~6s, unchanged
run uploads nothing, restore bit-identical, no files_external mount
created, no temp files (max 12 KiB in /tmp). Works with files_external
disabled when its classes are loaded. Backup peak memory 85-89 MiB on
these targets (to investigate).
This commit is contained in:
Dillard Blom
2026-10-01 18:33:15 +02:00
parent 915e534ab2
commit bc16a186d5
8 changed files with 339 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Backend/ExternalStorageFactory.php
+73
View File
@@ -0,0 +1,73 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Backend;
use OCA\Files_External\Lib\StorageConfig;
use OCA\Files_External\Service\BackendService;
use OCP\Files\Storage\IStorage;
use OCP\IUser;
/**
* Builds a storage object with files_external's own backends and auth mechanisms, the same
* way files_external's ConfigAdapter does it, but from a configuration that is never saved as
* a mount. The backup location therefore never appears in Files, for anyone.
*
* files_external classes are internal API: everything that touches them is in this class.
*/
final class ExternalStorageFactory {
public function __construct(
private BackendService $backends,
) {
}
/**
* @return list<array{id:string, name:string, auth:list<string>, parameters:array<string, mixed>}>
* available backends, to build the settings form from
*/
public function describeBackends(): array {
$result = [];
foreach ($this->backends->getAvailableBackends() as $backend) {
$result[] = [
'id' => $backend->getIdentifier(),
'name' => $backend->getText(),
'auth' => array_keys($backend->getAuthSchemes()),
'parameters' => array_map(fn ($p) => $p->jsonSerialize(), $backend->getParameters()),
];
}
return $result;
}
/**
* @param array<string, mixed> $options backend and auth options (e.g. host, bucket, key, secret)
* @param IUser|null $admin user for auth mechanisms that need one (password/session-based ones are not supported)
*/
public function create(string $backendId, string $authId, array $options, ?IUser $admin = null): IStorage {
$backend = $this->backends->getBackend($backendId);
$auth = $this->backends->getAuthMechanism($authId);
if ($backend === null || $auth === null) {
throw new BackendException("Unknown storage backend or auth mechanism: $backendId / $authId");
}
if (!in_array($auth->getScheme(), array_keys($backend->getAuthSchemes()), true)) {
throw new BackendException("Auth mechanism $authId does not fit backend $backendId");
}
$config = new StorageConfig();
$config->setBackend($backend);
$config->setAuthMechanism($auth);
$config->setBackendOptions($options);
$config->setMountPoint('/ng_backup'); // never mounted, but some code expects a value
$auth->manipulateStorageConfig($config, $admin);
$backend->manipulateStorageConfig($config, $admin);
$class = $backend->getStorageClass();
$storage = new $class($config->getBackendOptions());
$storage = $backend->wrapStorage($storage);
return $auth->wrapStorage($storage);
}
}
lib/Backend/IBackend.php
+5 -1
View File
@@ -12,7 +12,11 @@ namespace OCA\NgBackup\Backend;
* Implementations: a local directory (spike, tests), and adapters over files_external storages.
*/
interface IBackend {
/** Store an object; the stream is read to EOF. Must not leave a partial object behind on failure. */
/**
* Store an object; the stream is read to EOF. Takes ownership of the stream: it is always
* closed afterwards (some storages, like files_external's S3, close it themselves).
* Must not leave a partial object behind on failure.
*/
public function put(string $path, $stream): void;
/** @return resource readable stream of the whole object */
lib/Backend/LocalBackend.php
+3
View File
@@ -35,6 +35,9 @@ final class LocalBackend implements IBackend {
}
} finally {
fclose($out);
if (is_resource($stream)) {
fclose($stream);
}
}
if (!rename($tmp, $target)) {
@unlink($tmp);
lib/Backend/StorageBackend.php
+148
View File
@@ -0,0 +1,148 @@
<?php
declare(strict_types=1);
// SPDX-FileCopyrightText: 2026 Dillard Blom
// SPDX-License-Identifier: AGPL-3.0-or-later
namespace OCA\NgBackup\Backend;
use OCP\Files\Storage\IStorage;
use OCP\Files\Storage\IWriteStreamStorage;
/**
* IBackend on top of a Nextcloud storage object (from files_external or core), below a base
* directory on that storage.
*/
final class StorageBackend implements IBackend {
/** Object stores write whole objects atomically; other storages get a temp name + rename. */
private bool $atomicPut;
public function __construct(
private IStorage $storage,
private string $base = '',
) {
$this->base = trim($base, '/');
$this->atomicPut = $storage->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class);
if ($this->base !== '') {
$this->mkdirs($this->base);
}
}
public function put(string $path, $stream): void {
$target = $this->abs($path);
$this->mkdirs(dirname($target));
$writeTo = $this->atomicPut ? $target : $target . '.part-' . bin2hex(random_bytes(4));
try {
if ($this->storage->instanceOfStorage(IWriteStreamStorage::class)) {
/** @var IWriteStreamStorage $s */
$s = $this->storage;
$s->writeStream($writeTo, $stream);
} else {
$out = $this->storage->fopen($writeTo, 'w');
if ($out === false || stream_copy_to_stream($stream, $out) === false) {
throw new BackendException('Write failed for ' . $path);
}
fclose($out);
}
if (is_resource($stream)) {
fclose($stream);
}
if ($writeTo !== $target && !$this->storage->rename($writeTo, $target)) {
throw new BackendException('Cannot finalise ' . $path);
}
} catch (\Throwable $e) {
if (is_resource($stream)) {
fclose($stream);
}
if ($writeTo !== $target) {
$this->storage->unlink($writeTo);
}
throw $e instanceof BackendException ? $e : new BackendException('Write failed for ' . $path . ': ' . $e->getMessage(), 0, $e);
}
}
public function get(string $path) {
$fh = $this->storage->fopen($this->abs($path), 'r');
if ($fh === false) {
throw new BackendException('Not found: ' . $path);
}
return $fh;
}
public function getRange(string $path, int $offset, int $length): string {
$fh = $this->get($path);
try {
if (fseek($fh, $offset) !== 0) {
throw new BackendException('Seek failed in ' . $path);
}
$data = '';
while (strlen($data) < $length && !feof($fh)) {
$chunk = fread($fh, $length - strlen($data));
if ($chunk === false) {
break;
}
$data .= $chunk;
}
if (strlen($data) !== $length) {
throw new BackendException('Short read in ' . $path);
}
return $data;
} finally {
fclose($fh);
}
}
public function exists(string $path): bool {
return $this->storage->file_exists($this->abs($path));
}
public function list(string $prefix): array {
$result = [];
$this->listInto($this->abs($prefix), $result);
sort($result);
$cut = $this->base === '' ? 0 : strlen($this->base) + 1;
return array_map(fn ($p) => substr($p, $cut), $result);
}
public function delete(string $path): void {
$this->storage->unlink($this->abs($path));
}
private function listInto(string $dir, array &$result): void {
if (!$this->storage->is_dir($dir)) {
return;
}
$dh = $this->storage->opendir($dir);
if ($dh === false) {
return;
}
while (($name = readdir($dh)) !== false) {
if ($name === '.' || $name === '..' || str_contains($name, '.part-')) {
continue;
}
$child = $dir . '/' . $name;
if ($this->storage->is_dir($child)) {
$this->listInto($child, $result);
} else {
$result[] = $child;
}
}
closedir($dh);
}
private function mkdirs(string $dir): void {
if ($dir === '' || $dir === '.' || $this->storage->is_dir($dir)) {
return;
}
$this->mkdirs(dirname($dir) === '.' ? '' : dirname($dir));
$this->storage->mkdir($dir);
}
private function abs(string $path): string {
if (str_contains($path, '..')) {
throw new BackendException('Invalid path: ' . $path);
}
return ltrim(($this->base === '' ? '' : $this->base . '/') . ltrim($path, '/'), '/');
}
}
lib/Repository/BlobIndex.php
-1
View File
@@ -61,7 +61,6 @@ final class BlobIndex {
fwrite($stream, $this->cipher->encryptString(json_encode($entries, JSON_THROW_ON_ERROR), 'index:' . $packId));
rewind($stream);
$this->backend->put('index/' . $packId, $stream);
fclose($stream);
foreach ($entries as $e) {
$this->blobs[$e['id']] = [$packId, $e['offset'], $e['length'], $e['raw'], $e['flags']];
unset($this->pending[$e['id']]);
lib/Repository/PackWriter.php
+2 -1
View File
@@ -59,7 +59,8 @@ final class PackWriter {
fwrite($this->buffer, $header . pack('N', strlen($header)));
rewind($this->buffer);
$path = 'packs/' . substr($packId, 0, 2) . '/' . $packId;
$this->backend->put($path, $this->buffer);
$this->backend->put($path, $this->buffer); // closes the buffer
unset($this->buffer);
$this->index->addPack($packId, $this->entries);
$this->reset();
}
lib/Repository/Repository.php
-2
View File
@@ -111,7 +111,6 @@ final class Repository {
rewind($encrypted);
$this->backend->put('snapshots/' . $snapshotId, $encrypted);
fclose($tree);
fclose($encrypted);
return ['snapshot' => $snapshotId] + $stats;
}
@@ -272,7 +271,6 @@ final class Repository {
fwrite($s, $data);
rewind($s);
$backend->put($path, $s);
fclose($s);
}
private static function readFull($fh, int $length): string {
spikes/04-external-storage.php
+108
View File
@@ -0,0 +1,108 @@
<?php
declare(strict_types=1);
// Spike D: repository on S3 and SFTP through files_external classes, without any mount.
// Run inside the Nextcloud container as the web user:
// php -d memory_limit=128M /var/www/html/custom_apps/ng_backup/spikes/04-external-storage.php [s3|sftp|both]
require '/var/www/html/lib/base.php';
set_exception_handler(null);
restore_error_handler();
require __DIR__ . '/bootstrap.php';
use OCA\Files_External\Service\BackendService;
use OCA\NgBackup\Backend\ExternalStorageFactory;
use OCA\NgBackup\Backend\StorageBackend;
use OCA\NgBackup\Repository\Repository;
\OC_App::loadApp('files_external');
$which = $argv[1] ?? 'both';
$ok = true;
$check = static function (string $name, bool $pass) use (&$ok): void {
printf("%-62s %s\n", $name, $pass ? 'PASS' : 'FAIL');
$ok = $ok && $pass;
};
$hashTree = static function (string $root): array {
$out = [];
$it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($root, FilesystemIterator::SKIP_DOTS));
foreach ($it as $f) {
if ($f->isFile()) {
$out[substr($f->getPathname(), strlen($root) + 1)] = hash_file('sha256', $f->getPathname());
}
}
ksort($out);
return $out;
};
// Source tree: 100 small files + one 120 MiB file.
$src = '/tmp/ngb-d-src';
exec('rm -rf /tmp/ngb-d-src /tmp/ngb-d-restore-*');
mkdir("$src/docs", 0700, true);
for ($i = 0; $i < 100; $i++) {
file_put_contents("$src/docs/note-$i.txt", str_repeat("notitie $i ", 2000));
}
$fh = fopen("$src/big.bin", 'wb');
for ($i = 0; $i < 120; $i++) {
fwrite($fh, random_bytes(1048576));
}
fclose($fh);
$state = $hashTree($src);
$factory = new ExternalStorageFactory(\OCP\Server::get(BackendService::class));
$names = array_column($factory->describeBackends(), 'id');
printf("files_external backends available: %s\n", implode(', ', $names));
$targets = [
's3' => ['amazons3', 'amazons3::accesskey', [
'bucket' => 'ngbackup', 'hostname' => 's3', 'port' => '8333', 'region' => 'us-east-1',
'use_ssl' => false, 'use_path_style' => true, 'autocreate' => true,
'key' => 'ngbtestkey', 'secret' => 'ngbtestsecret123',
]],
'sftp' => ['sftp', 'password::password', [
'host' => 'sftp', 'root' => '/repo', 'user' => 'ngb', 'password' => 'ngbtestpass',
]],
];
$mountsBefore = (int)\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()
->select($q = \OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()->func()->count('*'))
->from('external_mounts')->executeQuery()->fetchOne();
foreach ($targets as $name => [$backendId, $authId, $options]) {
if ($which !== 'both' && $which !== $name) {
continue;
}
echo "--- $name\n";
$storage = $factory->create($backendId, $authId, $options);
$backend = new StorageBackend($storage, 'spike-' . bin2hex(random_bytes(3)));
memory_reset_peak_usage();
$t = microtime(true);
$repo = Repository::init($backend, 'spike passphrase');
$r1 = $repo->backupDirectory($src, null, $name);
printf("backup: %.1fs, peak %.1f MiB, uploaded %.1f MiB\n", microtime(true) - $t, memory_get_peak_usage() / 1048576, $r1['uploaded'] / 1048576);
$t = microtime(true);
$r2 = $repo->backupDirectory($src, $r1['snapshot'], $name);
printf("unchanged run: %.2fs, uploaded %d bytes\n", microtime(true) - $t, $r2['uploaded']);
memory_reset_peak_usage();
$t = microtime(true);
$repo = Repository::open($backend, 'spike passphrase');
$repo->restore($r1['snapshot'], "/tmp/ngb-d-restore-$name");
printf("restore: %.1fs, peak %.1f MiB\n", microtime(true) - $t, memory_get_peak_usage() / 1048576);
$check("$name: restored tree identical", $hashTree("/tmp/ngb-d-restore-$name") === $state);
$check("$name: unchanged run uploads nothing", $r2['uploaded'] === 0);
$check("$name: peak memory < 64 MiB", memory_get_peak_usage() < 64 * 1048576);
$check("$name: objects on target: config, packs, index, snapshots", count($backend->list('snapshots')) === 2 && $backend->exists('config'));
}
$mountsAfter = (int)\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()
->select(\OCP\Server::get(\OCP\IDBConnection::class)->getQueryBuilder()->func()->count('*'))
->from('external_mounts')->executeQuery()->fetchOne();
$check("no files_external mount created ($mountsBefore -> $mountsAfter)", $mountsBefore === $mountsAfter);
exec('rm -rf /tmp/ngb-d-src /tmp/ngb-d-restore-*');
echo $ok ? "ALL PASS\n" : "FAILURES\n";
exit($ok ? 0 : 1);