petru/nextcloud-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #59494 from nextcloud/artonge/fix/drop_transaction_during_scans

Browse Source
This commit is contained in:
Benjamin Gaussorgues
2026-04-09 08:56:50 +02:00
committed by GitHub
parent 5b7ea4e858 680ddd93c2
commit c835064a6b
7 changed files with 97 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/files/lib/BackgroundJob/ScanFiles.php
+7 -2
View File
@@ -8,6 +8,7 @@
namespace OCA\Files\BackgroundJob;
use OC\Files\SetupManager;
use OC\Files\Utils\Scanner;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\TimedJob;
@@ -15,6 +16,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IUserManager;
use Psr\Log\LoggerInterface;
/**
@@ -33,6 +35,8 @@ class ScanFiles extends TimedJob {
private LoggerInterface $logger,
private IDBConnection $connection,
ITimeFactory $time,
private readonly SetupManager $setupManager,
private readonly IUserManager $userManager,
) {
parent::__construct($time);
// Run once per 10 minutes
@@ -42,10 +46,11 @@ class ScanFiles extends TimedJob {
protected function runScanner(string $user): void {
try {
$scanner = new Scanner(
$user,
$this->userManager->get($user),
null,
$this->dispatcher,
$this->logger
$this->logger,
$this->setupManager,
);
$scanner->backgroundScan('');
} catch (\Exception $e) {
apps/files/lib/Command/Scan.php
+6 -3
View File
@@ -11,6 +11,7 @@ use OC\Core\Command\Base;
use OC\Core\Command\InterruptedException;
use OC\DB\Connection;
use OC\DB\ConnectionAdapter;
use OC\Files\SetupManager;
use OC\Files\Storage\Wrapper\Jail;
use OC\Files\Utils\Scanner;
use OC\FilesMetadata\FilesMetadataManager;
@@ -49,6 +50,7 @@ class Scan extends Base {
private FilesMetadataManager $filesMetadataManager,
private IEventDispatcher $eventDispatcher,
private LoggerInterface $logger,
private SetupManager $setupManager,
) {
parent::__construct();
}
@@ -111,10 +113,11 @@ class Scan extends Base {
): void {
$connection = $this->reconnectToDatabase($output);
$scanner = new Scanner(
$user,
$this->userManager->get($user),
new ConnectionAdapter($connection),
Server::get(IEventDispatcher::class),
Server::get(LoggerInterface::class)
$this->eventDispatcher,
$this->logger,
$this->setupManager,
);
# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception
apps/files/lib/Command/ScanAppData.php
+2
View File
@@ -10,6 +10,7 @@ use OC\Core\Command\Base;
use OC\Core\Command\InterruptedException;
use OC\DB\Connection;
use OC\DB\ConnectionAdapter;
use OC\Files\SetupManager;
use OC\Files\Utils\Scanner;
use OC\ForbiddenException;
use OC\Preview\Storage\StorageFactory;
@@ -60,6 +61,7 @@ class ScanAppData extends Base {
new ConnectionAdapter($connection),
Server::get(IEventDispatcher::class),
Server::get(LoggerInterface::class),
Server::get(SetupManager::class),
);
}
apps/files/tests/BackgroundJob/ScanFilesTest.php
+5 -1
View File
@@ -9,6 +9,7 @@ declare(strict_types=1);
namespace OCA\Files\Tests\BackgroundJob;
use OC\Files\Mount\MountPoint;
use OC\Files\SetupManager;
use OC\Files\Storage\Temporary;
use OCA\Files\BackgroundJob\ScanFiles;
use OCP\AppFramework\Utility\ITimeFactory;
@@ -17,6 +18,7 @@ use OCP\Files\Config\IUserMountCache;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IUser;
use OCP\IUserManager;
use OCP\Server;
use Psr\Log\LoggerInterface;
use Test\TestCase;
@@ -51,7 +53,9 @@ class ScanFilesTest extends TestCase {
$dispatcher,
$logger,
$connection,
$this->createMock(ITimeFactory::class)
$this->createMock(ITimeFactory::class),
$this->createMock(SetupManager::class),
$this->createMock(IUserManager::class),
])
->onlyMethods(['runScanner'])
->getMock();
lib/private/Files/Utils/Scanner.php
+26 -66
View File
@@ -10,11 +10,11 @@ namespace OC\Files\Utils;
use OC\Files\Cache\Cache;
use OC\Files\Filesystem;
use OC\Files\Mount\MountPoint;
use OC\Files\SetupManager;
use OC\Files\Storage\FailedStorage;
use OC\Files\Storage\Home;
use OC\ForbiddenException;
use OC\Hooks\PublicEmitter;
use OC\Lock\DBLockingProvider;
use OCA\Files_Sharing\SharedStorage;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\Events\BeforeFileScannedEvent;
@@ -29,9 +29,8 @@ use OCP\Files\NotFoundException;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageNotAvailableException;
use OCP\IDBConnection;
use OCP\Lock\ILockingProvider;
use OCP\IUser;
use OCP\Lock\LockedException;
use OCP\Server;
use Psr\Log\LoggerInterface;
/**
@@ -44,26 +43,14 @@ use Psr\Log\LoggerInterface;
* @package OC\Files\Utils
*/
class Scanner extends PublicEmitter {
public const MAX_ENTRIES_TO_COMMIT = 10000;
/**
* Whether to use a DB transaction
*/
protected bool $useTransaction;
/**
* Number of entries scanned to commit
*/
protected int $entriesToCommit = 0;
public function __construct(
private ?string $user,
protected ?IDBConnection $db,
private IEventDispatcher $dispatcher,
protected LoggerInterface $logger,
private readonly ?IUser $user,
protected readonly ?IDBConnection $db,
private readonly IEventDispatcher $eventDispatcher,
protected readonly LoggerInterface $logger,
private readonly SetupManager $setupManager,
) {
// when DB locking is used, no DB transactions will be used
$this->useTransaction = !(Server::get(ILockingProvider::class) instanceof DBLockingProvider);
}
/**
@@ -74,8 +61,11 @@ class Scanner extends PublicEmitter {
*/
protected function getMounts($dir) {
//TODO: move to the node based fileapi once that's done
\OC_Util::tearDownFS();
\OC_Util::setupFS($this->user);
$this->setupManager->tearDown();
if ($this->user !== null) {
$this->setupManager->setupForUser($this->user);
}
$mountManager = Filesystem::getMountManager();
$mounts = $mountManager->findIn($dir);
@@ -88,37 +78,32 @@ class Scanner extends PublicEmitter {
/**
* attach listeners to the scanner
*
* @param MountPoint $mount
*/
protected function attachListener($mount) {
protected function attachListener(MountPoint $mount) {
/** @var \OC\Files\Cache\Scanner $scanner */
$scanner = $mount->getStorage()->getScanner();
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function ($path) use ($mount): void {
$this->emit('\OC\Files\Utils\Scanner', 'scanFile', [$mount->getMountPoint() . $path]);
$this->dispatcher->dispatchTyped(new BeforeFileScannedEvent($mount->getMountPoint() . $path));
$this->eventDispatcher->dispatchTyped(new BeforeFileScannedEvent($mount->getMountPoint() . $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount): void {
$this->emit('\OC\Files\Utils\Scanner', 'scanFolder', [$mount->getMountPoint() . $path]);
$this->dispatcher->dispatchTyped(new BeforeFolderScannedEvent($mount->getMountPoint() . $path));
$this->eventDispatcher->dispatchTyped(new BeforeFolderScannedEvent($mount->getMountPoint() . $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'postScanFile', function ($path) use ($mount): void {
$this->emit('\OC\Files\Utils\Scanner', 'postScanFile', [$mount->getMountPoint() . $path]);
$this->dispatcher->dispatchTyped(new FileScannedEvent($mount->getMountPoint() . $path));
$this->eventDispatcher->dispatchTyped(new FileScannedEvent($mount->getMountPoint() . $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'postScanFolder', function ($path) use ($mount): void {
$this->emit('\OC\Files\Utils\Scanner', 'postScanFolder', [$mount->getMountPoint() . $path]);
$this->dispatcher->dispatchTyped(new FolderScannedEvent($mount->getMountPoint() . $path));
$this->eventDispatcher->dispatchTyped(new FolderScannedEvent($mount->getMountPoint() . $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', function ($path) use ($mount): void {
$this->emit('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', [$path]);
});
}
/**
* @param string $dir
*/
public function backgroundScan($dir) {
public function backgroundScan(string $dir) {
$mounts = $this->getMounts($dir);
foreach ($mounts as $mount) {
try {
@@ -157,13 +142,10 @@ class Scanner extends PublicEmitter {
}
/**
* @param string $dir
* @param $recursive
* @param callable|null $mountFilter
* @throws ForbiddenException
* @throws NotFoundException
*/
public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECURSIVE, ?callable $mountFilter = null) {
public function scan(string $dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECURSIVE, ?callable $mountFilter = null) {
if (!Filesystem::isValidPath($dir)) {
throw new \InvalidArgumentException('Invalid path to scan');
}
@@ -214,23 +196,22 @@ class Scanner extends PublicEmitter {
$relativePath = $mount->getInternalPath($dir);
/** @var \OC\Files\Cache\Scanner $scanner */
$scanner = $storage->getScanner();
$scanner->setUseTransactions(false);
$this->attachListener($mount);
$scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage): void {
$this->postProcessEntry($storage, $path);
$this->dispatcher->dispatchTyped(new NodeRemovedFromCache($storage, $path));
$this->triggerPropagator($storage, $path);
$this->eventDispatcher->dispatchTyped(new NodeRemovedFromCache($storage, $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage): void {
$this->postProcessEntry($storage, $path);
$this->dispatcher->dispatchTyped(new FileCacheUpdated($storage, $path));
$this->triggerPropagator($storage, $path);
$this->eventDispatcher->dispatchTyped(new FileCacheUpdated($storage, $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path, $storageId, $data, $fileId) use ($storage): void {
$this->postProcessEntry($storage, $path);
$this->triggerPropagator($storage, $path);
if ($fileId) {
$this->dispatcher->dispatchTyped(new FileCacheUpdated($storage, $path));
$this->eventDispatcher->dispatchTyped(new FileCacheUpdated($storage, $path));
} else {
$this->dispatcher->dispatchTyped(new NodeAddedToCache($storage, $path));
$this->eventDispatcher->dispatchTyped(new NodeAddedToCache($storage, $path));
}
});
@@ -238,9 +219,6 @@ class Scanner extends PublicEmitter {
throw new NotFoundException($dir);
}
if ($this->useTransaction) {
$this->db->beginTransaction();
}
try {
$propagator = $storage->getPropagator();
$propagator->beginBatch();
@@ -263,28 +241,10 @@ class Scanner extends PublicEmitter {
$this->logger->error('Storage ' . $storage->getId() . ' not available', ['exception' => $e]);
$this->emit('\OC\Files\Utils\Scanner', 'StorageNotAvailable', [$e]);
}
if ($this->useTransaction) {
$this->db->commit();
}
}
}
private function triggerPropagator(IStorage $storage, $internalPath) {
$storage->getPropagator()->propagateChange($internalPath, time());
}
private function postProcessEntry(IStorage $storage, $internalPath) {
$this->triggerPropagator($storage, $internalPath);
if ($this->useTransaction) {
$this->entriesToCommit++;
if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) {
$propagator = $storage->getPropagator();
$this->entriesToCommit = 0;
$this->db->commit();
$propagator->commitBatch();
$this->db->beginTransaction();
$propagator->beginBatch();
}
}
}
}
tests/lib/Files/EtagTest.php
+8 -1
View File
@@ -9,6 +9,7 @@
namespace Test\Files;
use OC\Files\Filesystem;
use OC\Files\SetupManager;
use OC\Files\Utils\Scanner;
use OCA\Files_Sharing\AppInfo\Application;
use OCP\EventDispatcher\IEventDispatcher;
@@ -73,7 +74,13 @@ class EtagTest extends \Test\TestCase {
$files = ['/foo.txt', '/folder/bar.txt', '/folder/subfolder', '/folder/subfolder/qwerty.txt'];
$originalEtags = $this->getEtags($files);
$scanner = new Scanner($user1, Server::get(IDBConnection::class), Server::get(IEventDispatcher::class), Server::get(LoggerInterface::class));
$scanner = new Scanner(
Server::get(IUserManager::class)->get($user1),
Server::get(IDBConnection::class),
Server::get(IEventDispatcher::class),
Server::get(LoggerInterface::class),
Server::get(SetupManager::class),
);
$scanner->backgroundScan('/');
$newEtags = $this->getEtags($files);
tests/lib/Files/Utils/ScannerTest.php
+43 -6
View File
@@ -10,6 +10,7 @@ namespace Test\Files\Utils;
use OC\Files\Filesystem;
use OC\Files\Mount\MountPoint;
use OC\Files\SetupManager;
use OC\Files\Storage\Temporary;
use OC\Files\Utils\Scanner;
use OCP\EventDispatcher\IEventDispatcher;
@@ -77,7 +78,13 @@ class ScannerTest extends \Test\TestCase {
$storage->file_put_contents('foo.txt', 'qwerty');
$storage->file_put_contents('folder/bar.txt', 'qwerty');
$scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class));
$scanner = new TestScanner(
Server::get(IUserManager::class)->get(''),
Server::get(IDBConnection::class),
$this->createMock(IEventDispatcher::class),
Server::get(LoggerInterface::class),
Server::get(SetupManager::class),
);
$scanner->addMount($mount);
$scanner->scan('');
@@ -99,7 +106,13 @@ class ScannerTest extends \Test\TestCase {
$storage->file_put_contents('foo.txt', 'qwerty');
$storage->file_put_contents('folder/bar.txt', 'qwerty');
$scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class));
$scanner = new TestScanner(
Server::get(IUserManager::class)->get(''),
Server::get(IDBConnection::class),
$this->createMock(IEventDispatcher::class),
Server::get(LoggerInterface::class),
Server::get(SetupManager::class),
);
$scanner->addMount($mount);
$scanner->scan('');
@@ -137,7 +150,13 @@ class ScannerTest extends \Test\TestCase {
$storage->file_put_contents('foo.txt', 'qwerty');
$storage->file_put_contents('folder/bar.txt', 'qwerty');
$scanner = new Scanner($uid, Server::get(IDBConnection::class), Server::get(IEventDispatcher::class), Server::get(LoggerInterface::class));
$scanner = new Scanner(
Server::get(IUserManager::class)->get($uid),
Server::get(IDBConnection::class),
Server::get(IEventDispatcher::class),
Server::get(LoggerInterface::class),
Server::get(SetupManager::class),
);
$this->assertFalse($cache->inCache('folder/bar.txt'));
$scanner->scan('/' . $uid . '/files/foo');
@@ -166,7 +185,13 @@ class ScannerTest extends \Test\TestCase {
$this->expectException(\InvalidArgumentException::class);
$this->expectExceptionMessage('Invalid path to scan');
$scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class));
$scanner = new TestScanner(
Server::get(IUserManager::class)->get(''),
Server::get(IDBConnection::class),
$this->createMock(IEventDispatcher::class),
Server::get(LoggerInterface::class),
Server::get(SetupManager::class),
);
$scanner->scan($invalidPath);
}
@@ -180,7 +205,13 @@ class ScannerTest extends \Test\TestCase {
$storage->file_put_contents('folder/bar.txt', 'qwerty');
$storage->touch('folder/bar.txt', time() - 200);
$scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class));
$scanner = new TestScanner(
Server::get(IUserManager::class)->get(''),
Server::get(IDBConnection::class),
$this->createMock(IEventDispatcher::class),
Server::get(LoggerInterface::class),
Server::get(SetupManager::class),
);
$scanner->addMount($mount);
$scanner->scan('');
@@ -206,7 +237,13 @@ class ScannerTest extends \Test\TestCase {
$storage->file_put_contents('folder/bar.txt', 'qwerty');
$storage->file_put_contents('folder/subfolder/foobar.txt', 'qwerty');
$scanner = new TestScanner('', Server::get(IDBConnection::class), $this->createMock(IEventDispatcher::class), Server::get(LoggerInterface::class));
$scanner = new TestScanner(
Server::get(IUserManager::class)->get(''),
Server::get(IDBConnection::class),
$this->createMock(IEventDispatcher::class),
Server::get(LoggerInterface::class),
Server::get(SetupManager::class),
);
$scanner->addMount($mount);
$scanner->scan('', $recusive = false);