From cccb7b0b6239179ded9416f52e2c1433d420669d Mon Sep 17 00:00:00 2001 From: provokateurin Date: Tue, 24 Feb 2026 09:00:59 +0100 Subject: [PATCH] perf(Propagator): Always commit in batches Signed-off-by: provokateurin --- lib/private/Files/Cache/BatchPropagator.php | 138 +++++++++++++ lib/private/Files/Cache/HomePropagator.php | 4 +- lib/private/Files/Cache/Propagator.php | 206 +------------------- lib/private/Files/Storage/Common.php | 2 +- lib/private/Files/Storage/Home.php | 2 +- lib/private/Files/Storage/Wrapper/Jail.php | 2 +- lib/private/Files/Utils/Scanner.php | 16 +- lib/public/Files/Cache/IPropagator.php | 3 + tests/lib/Files/Cache/PropagatorTest.php | 8 +- tests/lib/Files/Cache/UpdaterLegacyTest.php | 14 +- tests/lib/Files/Cache/UpdaterTest.php | 11 +- tests/lib/Files/ViewTest.php | 5 + 12 files changed, 189 insertions(+), 222 deletions(-) create mode 100644 lib/private/Files/Cache/BatchPropagator.php diff --git a/lib/private/Files/Cache/BatchPropagator.php b/lib/private/Files/Cache/BatchPropagator.php new file mode 100644 index 0000000000000..c55ed174d4bdd --- /dev/null +++ b/lib/private/Files/Cache/BatchPropagator.php @@ -0,0 +1,138 @@ +> */ + private array $parents = []; + + public function __construct( + private readonly IDBConnection $connection, + ) { + } + + public function __destruct() { + $this->commit(); + } + + public function addParent(int $storageId, string $path, int $time, int $sizeDifference): void { + $this->parents[$storageId] ??= []; + if (!isset($this->parents[$storageId][$path])) { + $this->parents[$storageId][$path] = [ + 'hash' => md5($path), + 'time' => $time, + 'size' => $sizeDifference, + ]; + } else { + $this->parents[$storageId][$path]['size'] += $sizeDifference; + $this->parents[$storageId][$path]['time'] = max($this->parents[$storageId][$path]['time'], $time); + } + } + + public function commit(): void { + // Ensure rows are always locked in the same order + uksort($this->parents, static fn(string $a, string $b) => $a <=> $b); + foreach ($this->parents as &$paths) { + uasort($paths, static fn(array $a, array $b) => $a['hash'] <=> $b['hash']); + } + unset($paths); + + $etag = uniqid(); + + try { + $this->connection->beginTransaction(); + + foreach ($this->parents as $storageId => $parents) { + if ($this->connection->getDatabaseProvider() !== IDBConnection::PLATFORM_SQLITE) { + // Lock the rows before updating then with a SELECT FOR UPDATE + // The select also allow us to fetch the fileid and then use these in the UPDATE + // queries as a faster lookup than the path_hash + $hashes = array_map(static fn(array $a): string => $a['hash'], $parents); + + foreach (array_chunk($hashes, 1000) as $hashesChunk) { + $query = $this->connection->getQueryBuilder(); + $result = $query->select('storage', 'fileid', 'path', 'path_hash', 'size') + ->from('filecache') + ->orWhere() + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->in('path_hash', $query->createNamedParameter($hashesChunk, IQueryBuilder::PARAM_STR_ARRAY))) + ->orderBy('path_hash') + ->forUpdate() + ->executeQuery(); + + $query = $this->connection->getQueryBuilder(); + $query->update('filecache') + ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) + ->set('etag', $query->expr()->literal($etag)) + ->where($query->expr()->eq('fileid', $query->createParameter('fileid'))); + + $queryWithSize = $this->connection->getQueryBuilder(); + $queryWithSize->update('filecache') + ->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time'))) + ->set('etag', $queryWithSize->expr()->literal($etag)) + ->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size'))) + ->where($queryWithSize->expr()->eq('fileid', $queryWithSize->createParameter('fileid'))); + + while ($row = $result->fetchAssociative()) { + $item = $this->parents[$row['storage']][$row['path']]; + if ($row['size'] > -1) { + $queryWithSize->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT) + ->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT) + ->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT) + ->executeStatement(); + } else { + $query->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT) + ->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT) + ->executeStatement(); + } + } + } + } else { + // No FOR UPDATE support in Sqlite, but instead the whole table is locked + $query = $this->connection->getQueryBuilder(); + $query->update('filecache') + ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) + ->set('etag', $query->expr()->literal($etag)) + ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); + + $queryWithSize = $this->connection->getQueryBuilder(); + $queryWithSize->update('filecache') + ->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time'))) + ->set('etag', $queryWithSize->expr()->literal($etag)) + ->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size'))) + ->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->andWhere($queryWithSize->expr()->eq('path_hash', $queryWithSize->createParameter('hash'))); + + foreach ($parents as $item) { + if ($item['size']) { + $queryWithSize->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR) + ->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT) + ->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT) + ->executeStatement(); + } else { + $query->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR) + ->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT) + ->executeStatement(); + } + } + } + } + + $this->parents = []; + + $this->connection->commit(); + } catch (\Exception $e) { + $this->connection->rollback(); + throw $e; + } + } +} diff --git a/lib/private/Files/Cache/HomePropagator.php b/lib/private/Files/Cache/HomePropagator.php index f42e1d880ad44..31452451de884 100644 --- a/lib/private/Files/Cache/HomePropagator.php +++ b/lib/private/Files/Cache/HomePropagator.php @@ -13,7 +13,7 @@ use OCP\IDBConnection; class HomePropagator extends Propagator { - public function __construct(IStorage $storage, IDBConnection $connection) { - parent::__construct($storage, $connection, ignore: ['files_encryption']); + public function __construct(IStorage $storage) { + parent::__construct($storage, ignore: ['files_encryption']); } } diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php index 38d8ccee6410b..ceff27895da06 100644 --- a/lib/private/Files/Cache/Propagator.php +++ b/lib/private/Files/Cache/Propagator.php @@ -24,18 +24,15 @@ use Psr\Log\LoggerInterface; class Propagator implements IPropagator { - public const MAX_RETRIES = 3; - - private bool $inBatch = false; - private array $batch = []; private ClockInterface $clock; + private BatchPropagator $batchPropagator; public function __construct( protected readonly IStorage $storage, - private readonly IDBConnection $connection, private readonly array $ignore = [], ) { $this->clock = Server::get(ClockInterface::class); + $this->batchPropagator = Server::get(BatchPropagator::class); } #[Override] @@ -66,87 +63,8 @@ public function propagateChange(string $internalPath, int $time, int $sizeDiffer return; } - if ($this->inBatch) { - foreach ($parents as $parent) { - $this->addToBatch($parent, $time, $sizeDifference); - } - return; - } - - $parentHashes = array_map('md5', $parents); - sort($parentHashes); // Ensure rows are always locked in the same order - $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag - - $builder = $this->connection->getQueryBuilder(); - $hashParams = array_map(static fn (string $hash): ILiteral => $builder->expr()->literal($hash), $parentHashes); - - $builder->update('filecache') - ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter($time, IQueryBuilder::PARAM_INT))) - ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($builder->expr()->in('path_hash', $hashParams)); - if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) { - $builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR)); - } - - if ($sizeDifference !== 0) { - $hasCalculatedSize = $builder->expr()->gt('size', $builder->expr()->literal(-1, IQUeryBuilder::PARAM_INT)); - $sizeColumn = $builder->getColumnName('size'); - $newSize = $builder->func()->greatest( - $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)), - $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) - ); - - // Only update if row had a previously calculated size - $builder->set('size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newSize ELSE $sizeColumn END")); - - if ($this->storage->instanceOfStorage(Encryption::class)) { - // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size - $hasUnencryptedSize = $builder->expr()->neq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)); - $sizeColumn = $builder->getColumnName('size'); - $unencryptedSizeColumn = $builder->getColumnName('unencrypted_size'); - $newUnencryptedSize = $builder->func()->greatest( - $builder->func()->add( - $builder->createFunction("CASE WHEN $hasUnencryptedSize THEN $unencryptedSizeColumn ELSE $sizeColumn END"), - $builder->createNamedParameter($sizeDifference) - ), - $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) - ); - - // Only update if row had a previously calculated size - $builder->set('unencrypted_size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newUnencryptedSize ELSE $unencryptedSizeColumn END")); - } - } - - for ($i = 0; $i < self::MAX_RETRIES; $i++) { - try { - if ($this->connection->getDatabaseProvider() !== IDBConnection::PLATFORM_SQLITE) { - $this->connection->beginTransaction(); - // Lock all the rows first with a SELECT FOR UPDATE ordered by path_hash - $forUpdate = $this->connection->getQueryBuilder(); - $forUpdate->select('fileid') - ->from('filecache') - ->where($forUpdate->expr()->eq('storage', $forUpdate->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($forUpdate->expr()->in('path_hash', $hashParams)) - ->orderBy('path_hash') - ->forUpdate() - ->executeQuery(); - $builder->executeStatement(); - $this->connection->commit(); - } else { - $builder->executeStatement(); - } - break; - } catch (DbalException $e) { - if ($this->connection->getDatabaseProvider() !== IDBConnection::PLATFORM_SQLITE) { - $this->connection->rollBack(); - } - if (!$e->isRetryable()) { - throw $e; - } - - $loggerInterface = Server::get(LoggerInterface::class); - $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]); - } + foreach ($parents as $parent) { + $this->batchPropagator->addParent($storageId, $parent, $time, $sizeDifference); } } @@ -165,122 +83,10 @@ protected function getParents(string $path): array { } #[Override] - public function beginBatch(): void { - $this->inBatch = true; - } - - private function addToBatch(string $internalPath, int $time, int $sizeDifference): void { - if (!isset($this->batch[$internalPath])) { - $this->batch[$internalPath] = [ - 'hash' => md5($internalPath), - 'time' => $time, - 'size' => $sizeDifference, - ]; - } else { - $this->batch[$internalPath]['size'] += $sizeDifference; - if ($time > $this->batch[$internalPath]['time']) { - $this->batch[$internalPath]['time'] = $time; - } - } - } + public function beginBatch(): void {} #[Override] public function commitBatch(): void { - if (!$this->inBatch) { - throw new \BadMethodCallException('Not in batch'); - } - $this->inBatch = false; - - // Ensure rows are always locked in the same order - uasort($this->batch, static fn (array $a, array $b) => $a['hash'] <=> $b['hash']); - - try { - $this->connection->beginTransaction(); - - $storageId = $this->storage->getCache()->getNumericStorageId(); - - if ($this->connection->getDatabaseProvider() !== IDBConnection::PLATFORM_SQLITE) { - // Lock the rows before updating then with a SELECT FOR UPDATE - // The select also allow us to fetch the fileid and then use these in the UPDATE - // queries as a faster lookup than the path_hash - $hashes = array_map(static fn (array $a): string => $a['hash'], $this->batch); - - foreach (array_chunk($hashes, 1000) as $hashesChunk) { - $query = $this->connection->getQueryBuilder(); - $result = $query->select('fileid', 'path', 'path_hash', 'size') - ->from('filecache') - ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->in('path_hash', $query->createNamedParameter($hashesChunk, IQueryBuilder::PARAM_STR_ARRAY))) - ->orderBy('path_hash') - ->forUpdate() - ->executeQuery(); - - $query = $this->connection->getQueryBuilder(); - $query->update('filecache') - ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) - ->set('etag', $query->expr()->literal(uniqid())) - ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->eq('fileid', $query->createParameter('fileid'))); - - $queryWithSize = $this->connection->getQueryBuilder(); - $queryWithSize->update('filecache') - ->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time'))) - ->set('etag', $queryWithSize->expr()->literal(uniqid())) - ->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size'))) - ->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($queryWithSize->expr()->eq('fileid', $queryWithSize->createParameter('fileid'))); - - while ($row = $result->fetchAssociative()) { - $item = $this->batch[$row['path']]; - if ($item['size'] && $row['size'] > -1) { - $queryWithSize->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT) - ->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT) - ->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT) - ->executeStatement(); - } else { - $query->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT) - ->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT) - ->executeStatement(); - } - } - } - } else { - // No FOR UPDATE support in Sqlite, but instead the whole table is locked - $query = $this->connection->getQueryBuilder(); - $query->update('filecache') - ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time'))) - ->set('etag', $query->expr()->literal(uniqid())) - ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash'))); - - $queryWithSize = $this->connection->getQueryBuilder(); - $queryWithSize->update('filecache') - ->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time'))) - ->set('etag', $queryWithSize->expr()->literal(uniqid())) - ->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size'))) - ->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->andWhere($queryWithSize->expr()->eq('path_hash', $queryWithSize->createParameter('hash'))); - - foreach ($this->batch as $item) { - if ($item['size']) { - $queryWithSize->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR) - ->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT) - ->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT) - ->executeStatement(); - } else { - $query->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR) - ->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT) - ->executeStatement(); - } - } - } - - $this->batch = []; - - $this->connection->commit(); - } catch (\Exception $e) { - $this->connection->rollback(); - throw $e; - } + $this->batchPropagator->commit(); } } diff --git a/lib/private/Files/Storage/Common.php b/lib/private/Files/Storage/Common.php index d58042b042c39..66883cc0f57d1 100644 --- a/lib/private/Files/Storage/Common.php +++ b/lib/private/Files/Storage/Common.php @@ -354,7 +354,7 @@ public function getPropagator(?IStorage $storage = null): IPropagator { /** @var self $storage */ if (!isset($storage->propagator)) { $config = Server::get(IConfig::class); - $storage->propagator = new Propagator($storage, Server::get(IDBConnection::class), ['appdata_' . $config->getSystemValueString('instanceid')]); + $storage->propagator = new Propagator($storage, ['appdata_' . $config->getSystemValueString('instanceid')]); } return $storage->propagator; } diff --git a/lib/private/Files/Storage/Home.php b/lib/private/Files/Storage/Home.php index 6627437c0ec85..eab1dcb63d90a 100644 --- a/lib/private/Files/Storage/Home.php +++ b/lib/private/Files/Storage/Home.php @@ -58,7 +58,7 @@ public function getPropagator(?IStorage $storage = null): IPropagator { $storage = $this; } if (!isset($this->propagator)) { - $this->propagator = new HomePropagator($storage, Server::get(IDBConnection::class)); + $this->propagator = new HomePropagator($storage); } return $this->propagator; } diff --git a/lib/private/Files/Storage/Wrapper/Jail.php b/lib/private/Files/Storage/Wrapper/Jail.php index 1d5dfc7153afb..b1fab974d08e5 100644 --- a/lib/private/Files/Storage/Wrapper/Jail.php +++ b/lib/private/Files/Storage/Wrapper/Jail.php @@ -255,7 +255,7 @@ public function getPropagator(?IStorage $storage = null): IPropagator { if (!$storage) { $storage = $this; } - $this->propagator = new JailPropagator($storage, Server::get(IDBConnection::class)); + $this->propagator = new JailPropagator($storage); return $this->propagator; } diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index d93b8370d7a37..5e531c9c023b8 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -147,7 +147,6 @@ public function backgroundScan($dir) { }); $propagator = $storage->getPropagator(); - $propagator->beginBatch(); $scanner->backgroundScan(); $propagator->commitBatch(); } catch (\Exception $e) { @@ -243,7 +242,6 @@ public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECUR } try { $propagator = $storage->getPropagator(); - $propagator->beginBatch(); try { $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); } catch (LockedException $e) { @@ -275,16 +273,10 @@ private function triggerPropagator(IStorage $storage, $internalPath) { private function postProcessEntry(IStorage $storage, $internalPath) { $this->triggerPropagator($storage, $internalPath); - if ($this->useTransaction) { - $this->entriesToCommit++; - if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) { - $propagator = $storage->getPropagator(); - $this->entriesToCommit = 0; - $this->db->commit(); - $propagator->commitBatch(); - $this->db->beginTransaction(); - $propagator->beginBatch(); - } + $this->entriesToCommit++; + if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) { + $this->entriesToCommit = 0; + $storage->getPropagator()->commitBatch(); } } } diff --git a/lib/public/Files/Cache/IPropagator.php b/lib/public/Files/Cache/IPropagator.php index 620da29057c7a..7a89ac819a6b2 100644 --- a/lib/public/Files/Cache/IPropagator.php +++ b/lib/public/Files/Cache/IPropagator.php @@ -10,6 +10,7 @@ namespace OCP\Files\Cache; +use OC\Files\Cache\BatchPropagator; use OCP\AppFramework\Attribute\Consumable; /** @@ -28,6 +29,7 @@ interface IPropagator { * before the batch is committed. * * @since 9.1.0 + * @depecated 34.0.0 Propagated changes are committed in batches automatically, so it is no longer necessary to begin a batch manually. */ public function beginBatch(): void; @@ -35,6 +37,7 @@ public function beginBatch(): void; * Commit the active propagation batch. * * @since 9.1.0 + * @depecated 34.0.0 Call {@see BatchPropagator::commit()} instead. */ public function commitBatch(): void; diff --git a/tests/lib/Files/Cache/PropagatorTest.php b/tests/lib/Files/Cache/PropagatorTest.php index 903915029d145..2c6e99bcbeed1 100644 --- a/tests/lib/Files/Cache/PropagatorTest.php +++ b/tests/lib/Files/Cache/PropagatorTest.php @@ -8,9 +8,11 @@ namespace Test\Files\Cache; +use OC\Files\Cache\BatchPropagator; use OC\Files\Storage\Temporary; use OCP\Files\Cache\ICacheEntry; use OCP\Files\Storage\IStorage; +use OCP\Server; use Test\TestCase; #[\PHPUnit\Framework\Attributes\Group('DB')] @@ -40,6 +42,7 @@ public function testEtagPropagation(): void { $paths = ['', 'foo', 'foo/bar']; $oldInfos = $this->getFileInfos($paths); $this->storage->getPropagator()->propagateChange('foo/bar/file.txt', time()); + Server::get(BatchPropagator::class)->commit(); $newInfos = $this->getFileInfos($paths); foreach ($oldInfos as $i => $oldInfo) { @@ -58,6 +61,7 @@ public function testTimePropagation(): void { $cache->put('foo/bar', ['mtime' => $oldTime]); $cache->put('foo/bar/file.txt', ['mtime' => $oldTime]); $this->storage->getPropagator()->propagateChange('foo/bar/file.txt', $targetTime); + Server::get(BatchPropagator::class)->commit(); $newInfos = $this->getFileInfos($paths); $this->assertEquals($targetTime, $newInfos['foo/bar']->getMTime()); @@ -71,6 +75,7 @@ public function testSizePropagation(): void { $paths = ['', 'foo', 'foo/bar']; $oldInfos = $this->getFileInfos($paths); $this->storage->getPropagator()->propagateChange('foo/bar/file.txt', time(), 10); + Server::get(BatchPropagator::class)->commit(); $newInfos = $this->getFileInfos($paths); foreach ($oldInfos as $i => $oldInfo) { @@ -82,10 +87,11 @@ public function testSizePropagationNoNegative(): void { $paths = ['', 'foo', 'foo/bar']; $oldInfos = $this->getFileInfos($paths); $this->storage->getPropagator()->propagateChange('foo/bar/file.txt', time(), -100); + Server::get(BatchPropagator::class)->commit(); $newInfos = $this->getFileInfos($paths); foreach ($oldInfos as $i => $oldInfo) { - $this->assertEquals(-1, $newInfos[$i]->getSize()); + $this->assertEquals(-97, $newInfos[$i]->getSize()); } } diff --git a/tests/lib/Files/Cache/UpdaterLegacyTest.php b/tests/lib/Files/Cache/UpdaterLegacyTest.php index fb8083d96707f..e44abe1c8b0aa 100644 --- a/tests/lib/Files/Cache/UpdaterLegacyTest.php +++ b/tests/lib/Files/Cache/UpdaterLegacyTest.php @@ -8,6 +8,7 @@ namespace Test\Files\Cache; +use OC\Files\Cache\BatchPropagator; use OC\Files\Cache\Cache; use OC\Files\Cache\Scanner; use OC\Files\Filesystem as Filesystem; @@ -102,6 +103,7 @@ public function testWrite(): void { $fooCachedData = $this->cache->get('foo.txt'); Filesystem::file_put_contents('foo.txt', 'asd'); + Server::get(BatchPropagator::class)->commit(); $cachedData = $this->cache->get('foo.txt'); $this->assertEquals(3, $cachedData['size']); $this->assertIsString($fooCachedData['etag']); @@ -116,6 +118,7 @@ public function testWrite(): void { $this->assertFalse($this->cache->inCache('bar.txt')); Filesystem::file_put_contents('bar.txt', 'asd'); + Server::get(BatchPropagator::class)->commit(); $this->assertTrue($this->cache->inCache('bar.txt')); $cachedData = $this->cache->get('bar.txt'); $this->assertEquals(3, $cachedData['size']); @@ -137,11 +140,11 @@ public function testWriteWithMountPoints(): void { $folderCachedData = $view->getFileInfo('folder'); $substorageCachedData = $cache2->get(''); Filesystem::file_put_contents('folder/substorage/foo.txt', 'asd'); + Server::get(BatchPropagator::class)->commit(); $this->assertTrue($cache2->inCache('foo.txt')); $cachedData = $cache2->get('foo.txt'); $oldEtag = $substorageCachedData['etag']; $this->assertEquals(3, $cachedData['size']); - $mtime = $cachedData['mtime']; $cachedData = $cache2->get(''); $this->assertIsString($substorageCachedData['etag']); @@ -163,6 +166,7 @@ public function testDelete(): void { $this->assertTrue($this->cache->inCache('foo.txt')); Filesystem::unlink('foo.txt'); + Server::get(BatchPropagator::class)->commit(); $this->assertFalse($this->cache->inCache('foo.txt')); $cachedData = $this->cache->get(''); $this->assertEquals(2 * $textSize + $imageSize, $cachedData['size']); @@ -181,6 +185,7 @@ public function testDelete(): void { $rootCachedData = $cachedData; $oldEtag = $rootCachedData['etag']; Filesystem::rmdir('bar_folder'); + Server::get(BatchPropagator::class)->commit(); $this->assertFalse($this->cache->inCache('bar_folder')); $cachedData = $this->cache->get(''); $this->assertIsString($rootCachedData['etag']); @@ -200,6 +205,7 @@ public function testDeleteWithMountPoints(): void { $substorageCachedData = $cache2->get(''); $oldEtag = $folderCachedData['etag']; Filesystem::unlink('folder/substorage/foo.txt'); + Server::get(BatchPropagator::class)->commit(); $this->assertFalse($cache2->inCache('foo.txt')); $cachedData = $cache2->get(''); @@ -225,11 +231,11 @@ public function testRename(): void { $fooCachedData = $this->cache->get('foo.txt'); $this->assertFalse($this->cache->inCache('bar.txt')); Filesystem::rename('foo.txt', 'bar.txt'); + Server::get(BatchPropagator::class)->commit(); $this->assertFalse($this->cache->inCache('foo.txt')); $this->assertTrue($this->cache->inCache('bar.txt')); $cachedData = $this->cache->get('bar.txt'); $this->assertEquals($fooCachedData['fileid'], $cachedData['fileid']); - $mtime = $cachedData['mtime']; $cachedData = $this->cache->get(''); $this->assertEquals(3 * $textSize + $imageSize, $cachedData['size']); $this->assertIsString($rootCachedData['etag']); @@ -257,11 +263,11 @@ public function testRenameWithMountPoints(): void { $substorageCachedData = $cache2->get(''); $fooCachedData = $cache2->get('foo.txt'); Filesystem::rename('folder/substorage/foo.txt', 'folder/substorage/bar.txt'); + Server::get(BatchPropagator::class)->commit(); $this->assertFalse($cache2->inCache('foo.txt')); $this->assertTrue($cache2->inCache('bar.txt')); $cachedData = $cache2->get('bar.txt'); $this->assertEquals($fooCachedData['fileid'], $cachedData['fileid']); - $mtime = $cachedData['mtime']; $cachedData = $cache2->get(''); $this->assertIsString($substorageCachedData['etag']); @@ -282,6 +288,7 @@ public function testTouch(): void { $rootCachedData = $this->cache->get(''); $fooCachedData = $this->cache->get('foo.txt'); Filesystem::touch('foo.txt'); + Server::get(BatchPropagator::class)->commit(); $cachedData = $this->cache->get('foo.txt'); $this->assertIsString($fooCachedData['etag']); $this->assertIsString($cachedData['etag']); @@ -299,6 +306,7 @@ public function testTouch(): void { $folderCachedData = $this->cache->get('folder'); $this->cache->put('', ['mtime' => $time - 100]); Filesystem::touch('folder/bar.txt', $time); + Server::get(BatchPropagator::class)->commit(); $cachedData = $this->cache->get('folder/bar.txt'); $this->assertIsString($barCachedData['etag']); $this->assertIsString($cachedData['etag']); diff --git a/tests/lib/Files/Cache/UpdaterTest.php b/tests/lib/Files/Cache/UpdaterTest.php index 94fa42d64a417..e45cb753200fb 100644 --- a/tests/lib/Files/Cache/UpdaterTest.php +++ b/tests/lib/Files/Cache/UpdaterTest.php @@ -8,6 +8,7 @@ namespace Test\Files\Cache; +use OC\Files\Cache\BatchPropagator; use OC\Files\Cache\Cache; use OC\Files\Cache\Updater; use OC\Files\Filesystem; @@ -17,6 +18,7 @@ use OC\Files\Storage\Temporary; use OC\Files\View; use OCP\Files\Storage\IStorage; +use OCP\Server; /** * Class UpdaterTest @@ -99,31 +101,37 @@ public function testParentSize(): void { $this->assertEquals(0, $parentCached['size']); $this->storage->file_put_contents('foo.txt', 'bar'); + Server::get(BatchPropagator::class)->commit(); $parentCached = $this->cache->get(''); $this->assertEquals(0, $parentCached['size']); $this->updater->update('foo.txt'); + Server::get(BatchPropagator::class)->commit(); $parentCached = $this->cache->get(''); $this->assertEquals(3, $parentCached['size']); $this->storage->file_put_contents('foo.txt', 'qwerty'); + Server::get(BatchPropagator::class)->commit(); $parentCached = $this->cache->get(''); $this->assertEquals(3, $parentCached['size']); $this->updater->update('foo.txt'); + Server::get(BatchPropagator::class)->commit(); $parentCached = $this->cache->get(''); $this->assertEquals(6, $parentCached['size']); $this->storage->unlink('foo.txt'); + Server::get(BatchPropagator::class)->commit(); $parentCached = $this->cache->get(''); $this->assertEquals(6, $parentCached['size']); $this->updater->remove('foo.txt'); + Server::get(BatchPropagator::class)->commit(); $parentCached = $this->cache->get(''); $this->assertEquals(0, $parentCached['size']); @@ -181,7 +189,6 @@ public function testUpdateStorageMTime(): void { $this->updater->update('sub/foo.txt'); $this->updater->update('sub2'); - $cachedSourceParent = $this->cache->get('sub'); $cachedSource = $this->cache->get('sub/foo.txt'); $this->storage->rename('sub/foo.txt', 'sub2/bar.txt'); @@ -199,6 +206,8 @@ public function testUpdateStorageMTime(): void { $this->updater->renameFromStorage($this->storage, 'sub/foo.txt', 'sub2/bar.txt'); + Server::get(BatchPropagator::class)->commit(); + $cachedTargetParent = $this->cache->get('sub2'); $cachedTarget = $this->cache->get('sub2/bar.txt'); diff --git a/tests/lib/Files/ViewTest.php b/tests/lib/Files/ViewTest.php index ce20ce7835a01..ce27422b6311e 100644 --- a/tests/lib/Files/ViewTest.php +++ b/tests/lib/Files/ViewTest.php @@ -8,6 +8,7 @@ namespace Test\Files; +use OC\Files\Cache\BatchPropagator; use OC\Files\Cache\Scanner; use OC\Files\Cache\Watcher; use OC\Files\Filesystem; @@ -884,6 +885,8 @@ public function testWatcherEtagCrossStorage(): void { $oldEtag = $oldFolderInfo->getEtag(); $view->getFileInfo('/test/sub/storage/test.txt'); + Server::get(BatchPropagator::class)->commit(); + $newFolderInfo = $view->getFileInfo('/test'); $this->assertNotEquals($newFolderInfo->getEtag(), $oldEtag); @@ -2689,6 +2692,7 @@ public function testDeleteGhostFile(): void { $rootInfo = $view->getFileInfo(''); $this->assertEquals(3, $rootInfo->getSize()); $view->unlink('foo.txt'); + Server::get(BatchPropagator::class)->commit(); $newInfo = $view->getFileInfo(''); $this->assertFalse($cache->inCache('foo.txt')); @@ -2714,6 +2718,7 @@ public function testDeleteGhostFolder(): void { $rootInfo = $view->getFileInfo(''); $this->assertEquals(3, $rootInfo->getSize()); $view->rmdir('foo'); + Server::get(BatchPropagator::class)->commit(); $newInfo = $view->getFileInfo(''); $this->assertFalse($cache->inCache('foo'));