From 72812b2b07cb27376605df03bab9385325e171f4 Mon Sep 17 00:00:00 2001 From: Louis Chmn Date: Wed, 8 Apr 2026 10:30:36 +0200 Subject: [PATCH 1/2] chore(Scanner): Use modern syntax and APIs Signed-off-by: Louis Chmn --- apps/files/lib/BackgroundJob/ScanFiles.php | 9 +++- apps/files/lib/Command/Scan.php | 9 ++-- apps/files/lib/Command/ScanAppData.php | 2 + .../tests/BackgroundJob/ScanFilesTest.php | 6 ++- lib/private/Files/Utils/Scanner.php | 48 +++++++++--------- tests/lib/Files/EtagTest.php | 9 +++- tests/lib/Files/Utils/ScannerTest.php | 49 ++++++++++++++++--- 7 files changed, 94 insertions(+), 38 deletions(-) diff --git a/apps/files/lib/BackgroundJob/ScanFiles.php b/apps/files/lib/BackgroundJob/ScanFiles.php index d1c39d2c2944b..da178f4c3699a 100644 --- a/apps/files/lib/BackgroundJob/ScanFiles.php +++ b/apps/files/lib/BackgroundJob/ScanFiles.php @@ -8,6 +8,7 @@ namespace OCA\Files\BackgroundJob; +use OC\Files\SetupManager; use OC\Files\Utils\Scanner; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; @@ -15,6 +16,7 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\IConfig; use OCP\IDBConnection; +use OCP\IUserManager; use Psr\Log\LoggerInterface; /** @@ -33,6 +35,8 @@ public function __construct( private LoggerInterface $logger, private IDBConnection $connection, ITimeFactory $time, + private readonly SetupManager $setupManager, + private readonly IUserManager $userManager, ) { parent::__construct($time); // Run once per 10 minutes @@ -42,10 +46,11 @@ public function __construct( protected function runScanner(string $user): void { try { $scanner = new Scanner( - $user, + $this->userManager->get($user), null, $this->dispatcher, - $this->logger + $this->logger, + $this->setupManager, ); $scanner->backgroundScan(''); } catch (\Exception $e) { diff --git a/apps/files/lib/Command/Scan.php b/apps/files/lib/Command/Scan.php index b9057139b0e16..71582e00c6e03 100644 --- a/apps/files/lib/Command/Scan.php +++ b/apps/files/lib/Command/Scan.php @@ -11,6 +11,7 @@ use OC\Core\Command\InterruptedException; use OC\DB\Connection; use OC\DB\ConnectionAdapter; +use OC\Files\SetupManager; use OC\Files\Storage\Wrapper\Jail; use OC\Files\Utils\Scanner; use OC\FilesMetadata\FilesMetadataManager; @@ -49,6 +50,7 @@ public function __construct( private FilesMetadataManager $filesMetadataManager, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, + private SetupManager $setupManager, ) { parent::__construct(); } @@ -111,10 +113,11 @@ protected function scanFiles( ): void { $connection = $this->reconnectToDatabase($output); $scanner = new Scanner( - $user, + $this->userManager->get($user), new ConnectionAdapter($connection), - Server::get(IEventDispatcher::class), - Server::get(LoggerInterface::class) + $this->eventDispatcher, + $this->logger, + $this->setupManager, ); # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index ade5e10fe68c2..ac9e0630dbd2c 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -10,6 +10,7 @@ use OC\Core\Command\InterruptedException; use OC\DB\Connection; use OC\DB\ConnectionAdapter; +use OC\Files\SetupManager; use OC\Files\Utils\Scanner; use OC\ForbiddenException; use OC\Preview\Storage\StorageFactory; @@ -60,6 +61,7 @@ protected function getScanner(OutputInterface $output): Scanner { new ConnectionAdapter($connection), Server::get(IEventDispatcher::class), Server::get(LoggerInterface::class), + Server::get(SetupManager::class), ); } diff --git a/apps/files/tests/BackgroundJob/ScanFilesTest.php b/apps/files/tests/BackgroundJob/ScanFilesTest.php index ed9e0b7767b86..d6f19ff7f0c71 100644 --- a/apps/files/tests/BackgroundJob/ScanFilesTest.php +++ b/apps/files/tests/BackgroundJob/ScanFilesTest.php @@ -9,6 +9,7 @@ namespace OCA\Files\Tests\BackgroundJob; use OC\Files\Mount\MountPoint; +use OC\Files\SetupManager; use OC\Files\Storage\Temporary; use OCA\Files\BackgroundJob\ScanFiles; use OCP\AppFramework\Utility\ITimeFactory; @@ -17,6 +18,7 @@ use OCP\IConfig; use OCP\IDBConnection; use OCP\IUser; +use OCP\IUserManager; use OCP\Server; use Psr\Log\LoggerInterface; use Test\TestCase; @@ -51,7 +53,9 @@ protected function setUp(): void { $dispatcher, $logger, $connection, - $this->createMock(ITimeFactory::class) + $this->createMock(ITimeFactory::class), + $this->createMock(SetupManager::class), + $this->createMock(IUserManager::class), ]) ->onlyMethods(['runScanner']) ->getMock(); diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index d93b8370d7a37..31085cd8c9a5e 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -10,6 +10,7 @@ use OC\Files\Cache\Cache; use OC\Files\Filesystem; use OC\Files\Mount\MountPoint; +use OC\Files\SetupManager; use OC\Files\Storage\FailedStorage; use OC\Files\Storage\Home; use OC\ForbiddenException; @@ -29,6 +30,7 @@ use OCP\Files\Storage\IStorage; use OCP\Files\StorageNotAvailableException; use OCP\IDBConnection; +use OCP\IUser; use OCP\Lock\ILockingProvider; use OCP\Lock\LockedException; use OCP\Server; @@ -57,10 +59,11 @@ class Scanner extends PublicEmitter { protected int $entriesToCommit = 0; public function __construct( - private ?string $user, - protected ?IDBConnection $db, - private IEventDispatcher $dispatcher, - protected LoggerInterface $logger, + private readonly ?IUser $user, + protected readonly ?IDBConnection $db, + private readonly IEventDispatcher $eventDispatcher, + protected readonly LoggerInterface $logger, + private readonly SetupManager $setupManager, ) { // when DB locking is used, no DB transactions will be used $this->useTransaction = !(Server::get(ILockingProvider::class) instanceof DBLockingProvider); @@ -74,8 +77,11 @@ public function __construct( */ protected function getMounts($dir) { //TODO: move to the node based fileapi once that's done - \OC_Util::tearDownFS(); - \OC_Util::setupFS($this->user); + $this->setupManager->tearDown(); + + if ($this->user !== null) { + $this->setupManager->setupForUser($this->user); + } $mountManager = Filesystem::getMountManager(); $mounts = $mountManager->findIn($dir); @@ -88,37 +94,32 @@ protected function getMounts($dir) { /** * attach listeners to the scanner - * - * @param MountPoint $mount */ - protected function attachListener($mount) { + protected function attachListener(MountPoint $mount) { /** @var \OC\Files\Cache\Scanner $scanner */ $scanner = $mount->getStorage()->getScanner(); $scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function ($path) use ($mount): void { $this->emit('\OC\Files\Utils\Scanner', 'scanFile', [$mount->getMountPoint() . $path]); - $this->dispatcher->dispatchTyped(new BeforeFileScannedEvent($mount->getMountPoint() . $path)); + $this->eventDispatcher->dispatchTyped(new BeforeFileScannedEvent($mount->getMountPoint() . $path)); }); $scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount): void { $this->emit('\OC\Files\Utils\Scanner', 'scanFolder', [$mount->getMountPoint() . $path]); - $this->dispatcher->dispatchTyped(new BeforeFolderScannedEvent($mount->getMountPoint() . $path)); + $this->eventDispatcher->dispatchTyped(new BeforeFolderScannedEvent($mount->getMountPoint() . $path)); }); $scanner->listen('\OC\Files\Cache\Scanner', 'postScanFile', function ($path) use ($mount): void { $this->emit('\OC\Files\Utils\Scanner', 'postScanFile', [$mount->getMountPoint() . $path]); - $this->dispatcher->dispatchTyped(new FileScannedEvent($mount->getMountPoint() . $path)); + $this->eventDispatcher->dispatchTyped(new FileScannedEvent($mount->getMountPoint() . $path)); }); $scanner->listen('\OC\Files\Cache\Scanner', 'postScanFolder', function ($path) use ($mount): void { $this->emit('\OC\Files\Utils\Scanner', 'postScanFolder', [$mount->getMountPoint() . $path]); - $this->dispatcher->dispatchTyped(new FolderScannedEvent($mount->getMountPoint() . $path)); + $this->eventDispatcher->dispatchTyped(new FolderScannedEvent($mount->getMountPoint() . $path)); }); $scanner->listen('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', function ($path) use ($mount): void { $this->emit('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', [$path]); }); } - /** - * @param string $dir - */ - public function backgroundScan($dir) { + public function backgroundScan(string $dir) { $mounts = $this->getMounts($dir); foreach ($mounts as $mount) { try { @@ -157,13 +158,10 @@ public function backgroundScan($dir) { } /** - * @param string $dir - * @param $recursive - * @param callable|null $mountFilter * @throws ForbiddenException * @throws NotFoundException */ - public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECURSIVE, ?callable $mountFilter = null) { + public function scan(string $dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECURSIVE, ?callable $mountFilter = null) { if (!Filesystem::isValidPath($dir)) { throw new \InvalidArgumentException('Invalid path to scan'); } @@ -219,18 +217,18 @@ public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECUR $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage): void { $this->postProcessEntry($storage, $path); - $this->dispatcher->dispatchTyped(new NodeRemovedFromCache($storage, $path)); + $this->eventDispatcher->dispatchTyped(new NodeRemovedFromCache($storage, $path)); }); $scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage): void { $this->postProcessEntry($storage, $path); - $this->dispatcher->dispatchTyped(new FileCacheUpdated($storage, $path)); + $this->eventDispatcher->dispatchTyped(new FileCacheUpdated($storage, $path)); }); $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path, $storageId, $data, $fileId) use ($storage): void { $this->postProcessEntry($storage, $path); if ($fileId) { - $this->dispatcher->dispatchTyped(new FileCacheUpdated($storage, $path)); + $this->eventDispatcher->dispatchTyped(new FileCacheUpdated($storage, $path)); } else { - $this->dispatcher->dispatchTyped(new NodeAddedToCache($storage, $path)); + $this->eventDispatcher->dispatchTyped(new NodeAddedToCache($storage, $path)); } }); diff --git a/tests/lib/Files/EtagTest.php b/tests/lib/Files/EtagTest.php index 4fff936d83e83..d40a6b9ec364b 100644 --- a/tests/lib/Files/EtagTest.php +++ b/tests/lib/Files/EtagTest.php @@ -9,6 +9,7 @@ namespace Test\Files; use OC\Files\Filesystem; +use OC\Files\SetupManager; use OC\Files\Utils\Scanner; use OCA\Files_Sharing\AppInfo\Application; use OCP\EventDispatcher\IEventDispatcher; @@ -73,7 +74,13 @@ public function testNewUser(): void { $files = ['/foo.txt', '/folder/bar.txt', '/folder/subfolder', '/folder/subfolder/qwerty.txt']; $originalEtags = $this->getEtags($files); - $scanner = new Scanner($user1, Server::get(IDBConnection::class), Server::get(IEventDispatcher::class), Server::get(LoggerInterface::class)); + $scanner = new Scanner( + Server::get(IUserManager::class)->get($user1), + Server::get(IDBConnection::class), + Server::get(IEventDispatcher::class), + Server::get(LoggerInterface::class), + Server::get(SetupManager::class), + ); $scanner->backgroundScan('/'); $newEtags = $this->getEtags($files); diff --git a/tests/lib/Files/Utils/ScannerTest.php b/tests/lib/Files/Utils/ScannerTest.php index 3e60c0234cbc8..96d23cd5db877 100644 --- a/tests/lib/Files/Utils/ScannerTest.php +++ b/tests/lib/Files/Utils/ScannerTest.php @@ -10,6 +10,7 @@ use OC\Files\Filesystem; use OC\Files\Mount\MountPoint; +use OC\Files\SetupManager; use OC\Files\Storage\Temporary; use OC\Files\Utils\Scanner; use OCP\EventDispatcher\IEventDispatcher; @@ -77,7 +78,13 @@ public function testReuseExistingRoot(): void { $storage->file_put_contents('foo.txt', 'qwerty'); $storage->file_put_contents('folder/bar.txt', 'qwerty'); - $scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class)); + $scanner = new TestScanner( + Server::get(IUserManager::class)->get(''), + Server::get(IDBConnection::class), + $this->createMock(IEventDispatcher::class), + Server::get(LoggerInterface::class), + Server::get(SetupManager::class), + ); $scanner->addMount($mount); $scanner->scan(''); @@ -99,7 +106,13 @@ public function testReuseExistingFile(): void { $storage->file_put_contents('foo.txt', 'qwerty'); $storage->file_put_contents('folder/bar.txt', 'qwerty'); - $scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class)); + $scanner = new TestScanner( + Server::get(IUserManager::class)->get(''), + Server::get(IDBConnection::class), + $this->createMock(IEventDispatcher::class), + Server::get(LoggerInterface::class), + Server::get(SetupManager::class), + ); $scanner->addMount($mount); $scanner->scan(''); @@ -137,7 +150,13 @@ public function testScanSubMount(): void { $storage->file_put_contents('foo.txt', 'qwerty'); $storage->file_put_contents('folder/bar.txt', 'qwerty'); - $scanner = new Scanner($uid, Server::get(IDBConnection::class), Server::get(IEventDispatcher::class), Server::get(LoggerInterface::class)); + $scanner = new Scanner( + Server::get(IUserManager::class)->get($uid), + Server::get(IDBConnection::class), + Server::get(IEventDispatcher::class), + Server::get(LoggerInterface::class), + Server::get(SetupManager::class), + ); $this->assertFalse($cache->inCache('folder/bar.txt')); $scanner->scan('/' . $uid . '/files/foo'); @@ -166,7 +185,13 @@ public function testInvalidPathScanning($invalidPath): void { $this->expectException(\InvalidArgumentException::class); $this->expectExceptionMessage('Invalid path to scan'); - $scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class)); + $scanner = new TestScanner( + Server::get(IUserManager::class)->get(''), + Server::get(IDBConnection::class), + $this->createMock(IEventDispatcher::class), + Server::get(LoggerInterface::class), + Server::get(SetupManager::class), + ); $scanner->scan($invalidPath); } @@ -180,7 +205,13 @@ public function testPropagateEtag(): void { $storage->file_put_contents('folder/bar.txt', 'qwerty'); $storage->touch('folder/bar.txt', time() - 200); - $scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class)); + $scanner = new TestScanner( + Server::get(IUserManager::class)->get(''), + Server::get(IDBConnection::class), + $this->createMock(IEventDispatcher::class), + Server::get(LoggerInterface::class), + Server::get(SetupManager::class), + ); $scanner->addMount($mount); $scanner->scan(''); @@ -206,7 +237,13 @@ public function testShallow(): void { $storage->file_put_contents('folder/bar.txt', 'qwerty'); $storage->file_put_contents('folder/subfolder/foobar.txt', 'qwerty'); - $scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class)); + $scanner = new TestScanner( + Server::get(IUserManager::class)->get(''), + Server::get(IDBConnection::class), + $this->createMock(IEventDispatcher::class), + Server::get(LoggerInterface::class), + Server::get(SetupManager::class), + ); $scanner->addMount($mount); $scanner->scan('', $recusive = false); From 680ddd93c2de5a66590f3ec76d170a86768b6746 Mon Sep 17 00:00:00 2001 From: Louis Chmn Date: Wed, 8 Apr 2026 14:34:45 +0200 Subject: [PATCH 2/2] fix(Scanner): Remove high level transaction during scans Signed-off-by: Louis Chmn --- lib/private/Files/Utils/Scanner.php | 44 ++--------------------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index 31085cd8c9a5e..cca0b6784a211 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -15,7 +15,6 @@ use OC\Files\Storage\Home; use OC\ForbiddenException; use OC\Hooks\PublicEmitter; -use OC\Lock\DBLockingProvider; use OCA\Files_Sharing\SharedStorage; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Events\BeforeFileScannedEvent; @@ -31,9 +30,7 @@ use OCP\Files\StorageNotAvailableException; use OCP\IDBConnection; use OCP\IUser; -use OCP\Lock\ILockingProvider; use OCP\Lock\LockedException; -use OCP\Server; use Psr\Log\LoggerInterface; /** @@ -46,17 +43,6 @@ * @package OC\Files\Utils */ class Scanner extends PublicEmitter { - public const MAX_ENTRIES_TO_COMMIT = 10000; - - /** - * Whether to use a DB transaction - */ - protected bool $useTransaction; - - /** - * Number of entries scanned to commit - */ - protected int $entriesToCommit = 0; public function __construct( private readonly ?IUser $user, @@ -65,8 +51,6 @@ public function __construct( protected readonly LoggerInterface $logger, private readonly SetupManager $setupManager, ) { - // when DB locking is used, no DB transactions will be used - $this->useTransaction = !(Server::get(ILockingProvider::class) instanceof DBLockingProvider); } /** @@ -212,19 +196,18 @@ public function scan(string $dir = '', $recursive = \OC\Files\Cache\Scanner::SCA $relativePath = $mount->getInternalPath($dir); /** @var \OC\Files\Cache\Scanner $scanner */ $scanner = $storage->getScanner(); - $scanner->setUseTransactions(false); $this->attachListener($mount); $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage): void { - $this->postProcessEntry($storage, $path); + $this->triggerPropagator($storage, $path); $this->eventDispatcher->dispatchTyped(new NodeRemovedFromCache($storage, $path)); }); $scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage): void { - $this->postProcessEntry($storage, $path); + $this->triggerPropagator($storage, $path); $this->eventDispatcher->dispatchTyped(new FileCacheUpdated($storage, $path)); }); $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path, $storageId, $data, $fileId) use ($storage): void { - $this->postProcessEntry($storage, $path); + $this->triggerPropagator($storage, $path); if ($fileId) { $this->eventDispatcher->dispatchTyped(new FileCacheUpdated($storage, $path)); } else { @@ -236,9 +219,6 @@ public function scan(string $dir = '', $recursive = \OC\Files\Cache\Scanner::SCA throw new NotFoundException($dir); } - if ($this->useTransaction) { - $this->db->beginTransaction(); - } try { $propagator = $storage->getPropagator(); $propagator->beginBatch(); @@ -261,28 +241,10 @@ public function scan(string $dir = '', $recursive = \OC\Files\Cache\Scanner::SCA $this->logger->error('Storage ' . $storage->getId() . ' not available', ['exception' => $e]); $this->emit('\OC\Files\Utils\Scanner', 'StorageNotAvailable', [$e]); } - if ($this->useTransaction) { - $this->db->commit(); - } } } private function triggerPropagator(IStorage $storage, $internalPath) { $storage->getPropagator()->propagateChange($internalPath, time()); } - - private function postProcessEntry(IStorage $storage, $internalPath) { - $this->triggerPropagator($storage, $internalPath); - if ($this->useTransaction) { - $this->entriesToCommit++; - if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) { - $propagator = $storage->getPropagator(); - $this->entriesToCommit = 0; - $this->db->commit(); - $propagator->commitBatch(); - $this->db->beginTransaction(); - $propagator->beginBatch(); - } - } - } }