feat: move file cache to a background job

Signed-off-by: Robin Appelman <robin@icewind.nl>

core/BackgroundJobs/FileCacheGcJob.php

@@ -0,0 +1,58 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OC\Core\BackgroundJobs;
+
+use OC\Cache\File;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\TimedJob;
+use OCP\IAppConfig;
+use OCP\IUserManager;
+use Psr\Log\LoggerInterface;
+
+class FileCacheGcJob extends TimedJob {
+	public function __construct(
+		ITimeFactory $time,
+		private readonly LoggerInterface $logger,
+		private readonly IAppConfig $appConfig,
+		private readonly IUserManager $userManager,
+	) {
+		parent::__construct($time);
+
+		$this->setTimeSensitivity(self::TIME_INSENSITIVE);
+		$this->setInterval(24 * 60 * 60);
+	}
+
+	protected function run(mixed $argument): void {
+		$offset = $this->appConfig->getValueInt('core', 'files_gc_offset');
+
+		$users = $this->userManager->getSeenUsers($offset);
+		$start = time();
+		$count = 0;
+		foreach ($users as $user) {
+			$cache = new File();
+			try {
+				$cache->gc($user);
+			} catch (\Exception $e) {
+				$this->logger->warning('Exception when running cache gc.', [
+					'app' => 'core',
+					'exception' => $e,
+				]);
+			}
+			$count++;
+			$now = time();
+
+			// almost time for the next job run, stop early and save our location
+			if ($now - $start > 23 * 60 * 60) {
+				$this->appConfig->setValueInt('core', 'files_gc_offset', $offset + $count);
+				return;
+			}
+		}
+		$this->appConfig->setValueInt('core', 'files_gc_offset', 0);
+	}
+}

lib/base.php

@@ -874,23 +874,6 @@ class OC {
 					$throttler = Server::get(IThrottler::class);
 					$throttler->resetDelay($request->getRemoteAddress(), 'login', ['user' => $uid]);
 				}
-
-				try {
-					$cache = new \OC\Cache\File();
-					$cache->gc();
-				} catch (\OC\ServerNotAvailableException $e) {
-					// not a GC exception, pass it on
-					throw $e;
-				} catch (\OC\ForbiddenException $e) {
-					// filesystem blocked for this request, ignore
-				} catch (\Exception $e) {
-					// a GC exception should not prevent users from using OC,
-					// so log the exception
-					Server::get(LoggerInterface::class)->warning('Exception when running cache gc.', [
-						'app' => 'core',
-						'exception' => $e,
-					]);
-				}
 			});
 		}
 	}

lib/composer/composer/autoload_classmap.php

@@ -1196,6 +1196,7 @@ return array(
     'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => $baseDir . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php',
     'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => $baseDir . '/core/BackgroundJobs/CheckForUserCertificates.php',
     'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
+    'OC\\Core\\BackgroundJobs\\FileCacheGcJob' => $baseDir . '/core/BackgroundJobs/FileCacheGcJob.php',
     'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php',
     'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
     'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php',
@@ -1903,6 +1904,7 @@ return array(
     'OC\\Repair\\NC29\\SanitizeAccountProperties' => $baseDir . '/lib/private/Repair/NC29/SanitizeAccountProperties.php',
     'OC\\Repair\\NC29\\SanitizeAccountPropertiesJob' => $baseDir . '/lib/private/Repair/NC29/SanitizeAccountPropertiesJob.php',
     'OC\\Repair\\NC30\\RemoveLegacyDatadirFile' => $baseDir . '/lib/private/Repair/NC30/RemoveLegacyDatadirFile.php',
+    'OC\\Repair\\NC32\\AddFileCacheGcBackgroundJob' => $baseDir . '/lib/private/Repair/NC32/AddFileCacheGcBackgroundJob.php',
     'OC\\Repair\\OldGroupMembershipShares' => $baseDir . '/lib/private/Repair/OldGroupMembershipShares.php',
     'OC\\Repair\\Owncloud\\CleanPreviews' => $baseDir . '/lib/private/Repair/Owncloud/CleanPreviews.php',
     'OC\\Repair\\Owncloud\\CleanPreviewsBackgroundJob' => $baseDir . '/lib/private/Repair/Owncloud/CleanPreviewsBackgroundJob.php',

lib/composer/composer/autoload_static.php

@@ -1245,6 +1245,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php',
         'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CheckForUserCertificates.php',
         'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
+        'OC\\Core\\BackgroundJobs\\FileCacheGcJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/FileCacheGcJob.php',
         'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php',
         'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
         'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php',
@@ -1952,6 +1953,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Repair\\NC29\\SanitizeAccountProperties' => __DIR__ . '/../../..' . '/lib/private/Repair/NC29/SanitizeAccountProperties.php',
         'OC\\Repair\\NC29\\SanitizeAccountPropertiesJob' => __DIR__ . '/../../..' . '/lib/private/Repair/NC29/SanitizeAccountPropertiesJob.php',
         'OC\\Repair\\NC30\\RemoveLegacyDatadirFile' => __DIR__ . '/../../..' . '/lib/private/Repair/NC30/RemoveLegacyDatadirFile.php',
+        'OC\\Repair\\NC32\\AddFileCacheGcBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/NC32/AddFileCacheGcBackgroundJob.php',
         'OC\\Repair\\OldGroupMembershipShares' => __DIR__ . '/../../..' . '/lib/private/Repair/OldGroupMembershipShares.php',
         'OC\\Repair\\Owncloud\\CleanPreviews' => __DIR__ . '/../../..' . '/lib/private/Repair/Owncloud/CleanPreviews.php',
         'OC\\Repair\\Owncloud\\CleanPreviewsBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/Owncloud/CleanPreviewsBackgroundJob.php',

lib/private/Cache/File.php

@@ -7,38 +7,47 @@
  */
 namespace OC\Cache;
 
-use OC\Files\Filesystem;
-use OC\Files\View;
+use OCP\Files\File as FileNode;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
 use OCP\ICache;
+use OCP\IUser;
 use OCP\IUserSession;
 use OCP\Security\ISecureRandom;
 use OCP\Server;
 use Psr\Log\LoggerInterface;
 
 class File implements ICache {
-	/** @var View */
-	protected $storage;
+	protected ?Folder $storage = null;
 
 	/**
-	 * Returns the cache storage for the logged in user
+	 * Returns the cache folder for the logged in user
 	 *
-	 * @return \OC\Files\View cache storage
+	 * @return Folder cache folder
 	 * @throws \OC\ForbiddenException
 	 * @throws \OC\User\NoUserException
 	 */
-	protected function getStorage() {
+	protected function getStorage(?IUser $user = null): Folder {
 		if ($this->storage !== null) {
 			return $this->storage;
 		}
-		$session = Server::get(IUserSession::class);
-		if ($session->isLoggedIn()) {
-			$rootView = new View();
-			$userId = $session->getUser()->getUID();
-			Filesystem::initMountPoints($userId);
-			if (!$rootView->file_exists('/' . $userId . '/cache')) {
-				$rootView->mkdir('/' . $userId . '/cache');
+		if (!$user) {
+			$session = Server::get(IUserSession::class);
+			$user = $session->getUser();
+		}
+		$rootFolder = Server::get(IRootFolder::class);
+		if ($user) {
+			$userId = $user->getUID();
+			try {
+				$cacheFolder = $rootFolder->get('/' . $userId . '/cache');
+				if (!$cacheFolder instanceof Folder) {
+					throw new \Exception('Cache folder is a file');
+				}
+			} catch (NotFoundException $e) {
+				$cacheFolder = $rootFolder->newFolder('/' . $userId . '/cache');
 			}
-			$this->storage = new View('/' . $userId . '/cache');
+			$this->storage = $cacheFolder;
 			return $this->storage;
 		} else {
 			Server::get(LoggerInterface::class)->error('Can\'t get cache storage, user not logged in', ['app' => 'core']);
@@ -52,27 +61,29 @@ class File implements ICache {
 	 * @throws \OC\ForbiddenException
 	 */
 	public function get($key) {
-		$result = null;
-		if ($this->hasKey($key)) {
-			$storage = $this->getStorage();
-			$result = $storage->file_get_contents($key);
+		$storage = $this->getStorage();
+		try {
+			/** @var FileNode $item */
+			$item = $storage->get($key);
+			return $item->getContent();
+		} catch (NotFoundException $e) {
+			return null;
 		}
-		return $result;
 	}
 
 	/**
 	 * Returns the size of the stored/cached data
 	 *
 	 * @param string $key
-	 * @return int
+	 * @return int|float
 	 */
 	public function size($key) {
-		$result = 0;
-		if ($this->hasKey($key)) {
-			$storage = $this->getStorage();
-			$result = $storage->filesize($key);
+		$storage = $this->getStorage();
+		try {
+			return $storage->get($key)->getSize();
+		} catch (NotFoundException $e) {
+			return 0;
 		}
-		return $result;
 	}
 
 	/**
@@ -94,14 +105,14 @@ class File implements ICache {
 		// use part file to prevent hasKey() to find the key
 		// while it is being written
 		$keyPart = $key . '.' . $uniqueId . '.part';
-		if ($storage && $storage->file_put_contents($keyPart, $value)) {
-			if ($ttl === 0) {
-				$ttl = 86400; // 60*60*24
-			}
-			$result = $storage->touch($keyPart, time() + $ttl);
-			$result &= $storage->rename($keyPart, $key);
+		$file = $storage->newFile($keyPart, $value);
+		if ($ttl === 0) {
+			$ttl = 86400; // 60*60*24
 		}
-		return $result;
+		$file->touch(time() + $ttl);
+		$file->move($storage->getFullPath($key));
+
+		return true;
 	}
 
 	/**
@@ -110,11 +121,7 @@ class File implements ICache {
 	 * @throws \OC\ForbiddenException
 	 */
 	public function hasKey($key) {
-		$storage = $this->getStorage();
-		if ($storage && $storage->is_file($key) && $storage->isReadable($key)) {
-			return true;
-		}
-		return false;
+		return $this->getStorage()->nodeExists($key);
 	}
 
 	/**
@@ -124,10 +131,12 @@ class File implements ICache {
 	 */
 	public function remove($key) {
 		$storage = $this->getStorage();
-		if (!$storage) {
+		try {
+			$storage->get($key)->delete();
+			return true;
+		} catch (NotFoundException $e) {
 			return false;
 		}
-		return $storage->unlink($key);
 	}
 
 	/**
@@ -137,14 +146,9 @@ class File implements ICache {
 	 */
 	public function clear($prefix = '') {
 		$storage = $this->getStorage();
-		if ($storage && $storage->is_dir('/')) {
-			$dh = $storage->opendir('/');
-			if (is_resource($dh)) {
-				while (($file = readdir($dh)) !== false) {
-					if ($file !== '.' && $file !== '..' && ($prefix === '' || str_starts_with($file, $prefix))) {
-						$storage->unlink('/' . $file);
-					}
-				}
+		foreach ($storage->getDirectoryListing() as $file) {
+			if ($prefix === '' || str_starts_with($file->getName(), $prefix)) {
+				$file->delete();
 			}
 		}
 		return true;
@@ -154,32 +158,24 @@ class File implements ICache {
 	 * Runs GC
 	 * @throws \OC\ForbiddenException
 	 */
-	public function gc() {
-		$storage = $this->getStorage();
-		if ($storage) {
-			// extra hour safety, in case of stray part chunks that take longer to write,
-			// because touch() is only called after the chunk was finished
-			$now = time() - 3600;
-			$dh = $storage->opendir('/');
-			if (!is_resource($dh)) {
-				return null;
-			}
-			while (($file = readdir($dh)) !== false) {
-				if ($file !== '.' && $file !== '..') {
-					try {
-						$mtime = $storage->filemtime('/' . $file);
-						if ($mtime < $now) {
-							$storage->unlink('/' . $file);
-						}
-					} catch (\OCP\Lock\LockedException $e) {
-						// ignore locked chunks
-						Server::get(LoggerInterface::class)->debug('Could not cleanup locked chunk "' . $file . '"', ['app' => 'core']);
-					} catch (\OCP\Files\ForbiddenException $e) {
-						Server::get(LoggerInterface::class)->debug('Could not cleanup forbidden chunk "' . $file . '"', ['app' => 'core']);
-					} catch (\OCP\Files\LockNotAcquiredException $e) {
-						Server::get(LoggerInterface::class)->debug('Could not cleanup locked chunk "' . $file . '"', ['app' => 'core']);
-					}
+	public function gc(?IUser $user = null) {
+		$storage = $this->getStorage($user);
+		// extra hour safety, in case of stray part chunks that take longer to write,
+		// because touch() is only called after the chunk was finished
+
+		$now = time() - 3600;
+		foreach ($storage->getDirectoryListing() as $file) {
+			try {
+				if ($file->getMTime() < $now) {
+					$file->delete();
 				}
+			} catch (\OCP\Lock\LockedException $e) {
+				// ignore locked chunks
+				Server::get(LoggerInterface::class)->debug('Could not cleanup locked chunk "' . $file->getName() . '"', ['app' => 'core']);
+			} catch (\OCP\Files\ForbiddenException $e) {
+				Server::get(LoggerInterface::class)->debug('Could not cleanup forbidden chunk "' . $file->getName() . '"', ['app' => 'core']);
+			} catch (\OCP\Files\LockNotAcquiredException $e) {
+				Server::get(LoggerInterface::class)->debug('Could not cleanup locked chunk "' . $file->getName() . '"', ['app' => 'core']);
 			}
 		}
 	}

lib/private/Repair/NC32/AddFileCacheGcBackgroundJob.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OC\Repair\NC32;
+
+use OC\Core\BackgroundJobs\FileCacheGcJob;
+use OCP\BackgroundJob\IJobList;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+
+class AddFileCacheGcBackgroundJob implements IRepairStep {
+	public function __construct(
+		private readonly IJobList $jobList,
+	) {
+	}
+
+	public function getName(): string {
+		return 'Add background job to cleanup file cache';
+	}
+
+	public function run(IOutput $output) {
+		$this->jobList->add(FileCacheGcJob::class);
+	}
+}

tests/lib/Cache/FileCacheTest.php

@@ -7,8 +7,8 @@
 
 namespace Test\Cache;
 
-use OC\Files\Storage\Local;
 use OCP\Files\Mount\IMountManager;
+use OCP\Lock\ILockingProvider;
 use Test\Traits\UserTrait;
 
 /**
@@ -33,10 +33,6 @@ class FileCacheTest extends TestCache {
 	 * @var \OC\Files\Storage\Storage
 	 * */
 	private $storage;
-	/**
-	 * @var \OC\Files\View
-	 * */
-	private $rootView;
 
 	public function skip() {
 		//$this->skipUnless(OC_User::isLoggedIn());
@@ -58,12 +54,8 @@ class FileCacheTest extends TestCache {
 		$manager = \OC::$server->get(IMountManager::class);
 		$manager->removeMount('/test');
 
-		$storage = new \OC\Files\Storage\Temporary([]);
-		\OC\Files\Filesystem::mount($storage, [], '/test/cache');
-
-		//set up the users dir
-		$this->rootView = new \OC\Files\View('');
-		$this->rootView->mkdir('/test');
+		$this->storage = new \OC\Files\Storage\Temporary([]);
+		\OC\Files\Filesystem::mount($this->storage, [], '/test/cache');
 
 		$this->instance = new \OC\Cache\File();
 
@@ -86,71 +78,45 @@ class FileCacheTest extends TestCache {
 		parent::tearDown();
 	}
 
-	private function setupMockStorage() {
-		$mockStorage = $this->getMockBuilder(Local::class)
-			->setMethods(['filemtime', 'unlink'])
-			->setConstructorArgs([['datadir' => \OC::$server->getTempManager()->getTemporaryFolder()]])
-			->getMock();
-
-		\OC\Files\Filesystem::mount($mockStorage, [], '/test/cache');
-
-		return $mockStorage;
-	}
-
 	public function testGarbageCollectOldKeys(): void {
-		$mockStorage = $this->setupMockStorage();
-
-		$mockStorage->expects($this->atLeastOnce())
-			->method('filemtime')
-			->willReturn(100);
-		$mockStorage->expects($this->once())
-			->method('unlink')
-			->with('key1')
-			->willReturn(true);
-
 		$this->instance->set('key1', 'value1');
+
+		$this->assertTrue($this->storage->file_exists('key1'));
+		$this->storage->getCache()->put('key1', ['mtime' => 100]);
+
 		$this->instance->gc();
+		$this->assertFalse($this->storage->file_exists('key1'));
 	}
 
 	public function testGarbageCollectLeaveRecentKeys(): void {
-		$mockStorage = $this->setupMockStorage();
-
-		$mockStorage->expects($this->atLeastOnce())
-			->method('filemtime')
-			->willReturn(time() + 3600);
-		$mockStorage->expects($this->never())
-			->method('unlink')
-			->with('key1');
 		$this->instance->set('key1', 'value1');
+
+		$this->assertTrue($this->storage->file_exists('key1'));
+		$this->storage->getCache()->put('key1', ['mtime' => time() + 3600]);
+
 		$this->instance->gc();
+
+		$this->assertTrue($this->storage->file_exists('key1'));
 	}
 
-	public function lockExceptionProvider() {
-		return [
-			[new \OCP\Lock\LockedException('key1')],
-			[new \OCP\Files\LockNotAcquiredException('key1', 1)],
-		];
-	}
-
-	/**
-	 * @dataProvider lockExceptionProvider
-	 */
-	public function testGarbageCollectIgnoreLockedKeys($testException): void {
-		$mockStorage = $this->setupMockStorage();
-
-		$mockStorage->expects($this->atLeastOnce())
-			->method('filemtime')
-			->willReturn(100);
-		$mockStorage->expects($this->atLeastOnce())
-			->method('unlink')
-			->will($this->onConsecutiveCalls(
-				$this->throwException($testException),
-				$this->returnValue(true)
-			));
+	public function testGarbageCollectIgnoreLockedKeys(): void {
+		$lockingProvider = \OC::$server->get(ILockingProvider::class);
 
 		$this->instance->set('key1', 'value1');
+		$this->storage->getCache()->put('key1', ['mtime' => 100]);
 		$this->instance->set('key2', 'value2');
+		$this->storage->getCache()->put('key2', ['mtime' => 100]);
+		$this->storage->acquireLock('key2', ILockingProvider::LOCK_SHARED, $lockingProvider);
+
+		$this->assertTrue($this->storage->file_exists('key1'));
+		$this->assertTrue($this->storage->file_exists('key2'));
 
 		$this->instance->gc();
+
+		$this->storage->releaseLock('key2', ILockingProvider::LOCK_SHARED, $lockingProvider);
+
+		$this->assertFalse($this->storage->file_exists('key1'));
+		$this->assertFalse($this->storage->file_exists('key2'));
+
 	}
 }