From 224b74098d3cb64ca95973921199069a97302748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 20 Jun 2017 11:35:49 +0200 Subject: [PATCH] no trip to the db is better than a fast trip to the db move to Cache class to avoid storage type issues fix tests don't bother caching partial clear cache when manually deleting stuff from filecache more caching add fixTypes() check if storage_mtime isset really check if storage_mtime isset fix undefined index, remove obsolete todo use correct var fix more aggressive caching remove unneeded classes clear cache in tests and repair steps encryption needs a separate PR only clear if cache was initialized fix namespace only clear if cache has been initialized, add deprecation as a big fat codesmell --- apps/files/appinfo/update.php | 3 + .../files/lib/Command/DeleteOrphanedFiles.php | 4 + .../DeleteOrphanedItemsJobTest.php | 48 ++-- .../lib/Command/CleanupRemoteStorages.php | 4 + .../Command/CleanupRemoteStoragesTest.php | 4 + apps/testing/lib/BigFileID.php | 4 + lib/private/Files/Cache/Cache.php | 218 +++++++++++------- lib/private/Files/Cache/Propagator.php | 8 + lib/private/Files/Cache/Storage.php | 3 + lib/private/Files/Type/Loader.php | 4 + lib/private/Repair/FillETags.php | 4 + lib/private/Repair/RepairMimeTypes.php | 10 + tests/lib/Files/Cache/ScannerTest.php | 7 + tests/lib/Files/Config/UserMountCacheTest.php | 7 + tests/lib/Repair/CleanTagsTest.php | 7 + .../lib/Share20/DefaultShareProviderTest.php | 7 + tests/lib/TestCase.php | 4 + 17 files changed, 244 insertions(+), 102 deletions(-) diff --git a/apps/files/appinfo/update.php b/apps/files/appinfo/update.php index a7ff93eb9633..119ec1ef2757 100644 --- a/apps/files/appinfo/update.php +++ b/apps/files/appinfo/update.php @@ -51,6 +51,9 @@ */ function owncloud_reset_encrypted_flag(\OCP\IDBConnection $conn) { $conn->executeUpdate('UPDATE `*PREFIX*filecache` SET `encrypted` = 0 WHERE `encrypted` = 1'); + if (isset(\OC\Files\Cache\Cache::$metaDataCache)) { + \OC\Files\Cache\Cache::$metaDataCache->clear(); + } } // Current version of ownCloud before the update is 8.1.0 or 8.2.0.(0-2) diff --git a/apps/files/lib/Command/DeleteOrphanedFiles.php b/apps/files/lib/Command/DeleteOrphanedFiles.php index 34390afb14bb..c7b4b158ad45 100644 --- a/apps/files/lib/Command/DeleteOrphanedFiles.php +++ b/apps/files/lib/Command/DeleteOrphanedFiles.php @@ -22,6 +22,7 @@ namespace OCA\Files\Command; +use OC\Files\Cache\Cache; use OCP\IDBConnection; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -75,6 +76,9 @@ public function execute(InputInterface $input, OutputInterface $output) { } $result->closeCursor(); } + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $output->writeln("$deletedEntries orphaned file cache entries deleted"); } diff --git a/apps/files/tests/BackgroundJob/DeleteOrphanedItemsJobTest.php b/apps/files/tests/BackgroundJob/DeleteOrphanedItemsJobTest.php index 7e1b66927a54..5046534a72fb 100644 --- a/apps/files/tests/BackgroundJob/DeleteOrphanedItemsJobTest.php +++ b/apps/files/tests/BackgroundJob/DeleteOrphanedItemsJobTest.php @@ -22,6 +22,7 @@ namespace OCA\Files\Tests\BackgroundJob; +use OC\Files\Cache\Cache; use OCA\Files\BackgroundJob\DeleteOrphanedItems; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -42,6 +43,17 @@ protected function setup() { $this->connection = \OC::$server->getDatabaseConnection(); } + protected function cleanup($fileId, $table){ + $query = $this->connection->getQueryBuilder(); + $query->delete('filecache') + ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) + ->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } + $this->cleanMapping($table); + } + protected function cleanMapping($table) { $query = $this->connection->getQueryBuilder(); $query->delete($table)->execute(); @@ -71,6 +83,9 @@ public function testClearSystemTagMappings() { 'path' => $query->createNamedParameter('apps/files/tests/deleteorphanedtagsjobtest.php'), 'path_hash' => $query->createNamedParameter(\md5('apps/files/tests/deleteorphanedtagsjobtest.php')), ])->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $fileId = $query->getLastInsertId(); // Existing file @@ -100,11 +115,7 @@ public function testClearSystemTagMappings() { $mapping = $this->getMappings('systemtag_object_mapping'); $this->assertCount(1, $mapping); - $query = $this->connection->getQueryBuilder(); - $query->delete('filecache') - ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) - ->execute(); - $this->cleanMapping('systemtag_object_mapping'); + $this->cleanup($fileId, 'systemtag_object_mapping'); } /** @@ -120,6 +131,9 @@ public function testClearUserTagMappings() { 'path' => $query->createNamedParameter('apps/files/tests/deleteorphanedtagsjobtest.php'), 'path_hash' => $query->createNamedParameter(\md5('apps/files/tests/deleteorphanedtagsjobtest.php')), ])->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $fileId = $query->getLastInsertId(); // Existing file @@ -149,11 +163,7 @@ public function testClearUserTagMappings() { $mapping = $this->getMappings('vcategory_to_object'); $this->assertCount(1, $mapping); - $query = $this->connection->getQueryBuilder(); - $query->delete('filecache') - ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) - ->execute(); - $this->cleanMapping('vcategory_to_object'); + $this->cleanup($fileId, 'vcategory_to_object'); } /** @@ -169,6 +179,9 @@ public function testClearComments() { 'path' => $query->createNamedParameter('apps/files/tests/deleteorphanedtagsjobtest.php'), 'path_hash' => $query->createNamedParameter(\md5('apps/files/tests/deleteorphanedtagsjobtest.php')), ])->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $fileId = $query->getLastInsertId(); // Existing file @@ -200,11 +213,7 @@ public function testClearComments() { $mapping = $this->getMappings('comments'); $this->assertCount(1, $mapping); - $query = $this->connection->getQueryBuilder(); - $query->delete('filecache') - ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) - ->execute(); - $this->cleanMapping('comments'); + $this->cleanup($fileId, 'comments'); } /** @@ -220,6 +229,9 @@ public function testClearCommentReadMarks() { 'path' => $query->createNamedParameter('apps/files/tests/deleteorphanedtagsjobtest.php'), 'path_hash' => $query->createNamedParameter(\md5('apps/files/tests/deleteorphanedtagsjobtest.php')), ])->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $fileId = $query->getLastInsertId(); // Existing file @@ -249,11 +261,7 @@ public function testClearCommentReadMarks() { $mapping = $this->getMappings('comments_read_markers'); $this->assertCount(1, $mapping); - $query = $this->connection->getQueryBuilder(); - $query->delete('filecache') - ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) - ->execute(); - $this->cleanMapping('comments_read_markers'); + $this->cleanup($fileId, 'comments_read_markers'); } } diff --git a/apps/files_sharing/lib/Command/CleanupRemoteStorages.php b/apps/files_sharing/lib/Command/CleanupRemoteStorages.php index 583f93ea5c89..7651289227a7 100644 --- a/apps/files_sharing/lib/Command/CleanupRemoteStorages.php +++ b/apps/files_sharing/lib/Command/CleanupRemoteStorages.php @@ -22,6 +22,7 @@ namespace OCA\Files_Sharing\Command; +use OC\Files\Cache\Cache; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\IDBConnection; use Symfony\Component\Console\Command\Command; @@ -129,6 +130,9 @@ public function deleteFiles ($numericId, OutputInterface $output) { ); $output->write("deleting files for storage $numericId ... "); $count = $queryBuilder->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $output->writeln("deleted $count"); } diff --git a/apps/files_sharing/tests/Command/CleanupRemoteStoragesTest.php b/apps/files_sharing/tests/Command/CleanupRemoteStoragesTest.php index adb120879aa3..6cfe60db6a06 100644 --- a/apps/files_sharing/tests/Command/CleanupRemoteStoragesTest.php +++ b/apps/files_sharing/tests/Command/CleanupRemoteStoragesTest.php @@ -22,6 +22,7 @@ namespace OCA\Files_Sharing\Tests\Command; +use OC\Files\Cache\Cache; use OCA\Files_Sharing\Command\CleanupRemoteStorages; use Test\TestCase; @@ -101,6 +102,9 @@ protected function setup() { $filesQuery->setParameter(2, \md5('file' . $i)); $filesQuery->execute(); } + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } } diff --git a/apps/testing/lib/BigFileID.php b/apps/testing/lib/BigFileID.php index 47bf3fcbfaef..d478e0cf1df9 100644 --- a/apps/testing/lib/BigFileID.php +++ b/apps/testing/lib/BigFileID.php @@ -21,6 +21,7 @@ namespace OCA\Testing; +use OC\Files\Cache\Cache; use OCP\IDBConnection; /** @@ -53,6 +54,9 @@ public function increaseFileIDsBeyondMax32bits(){ '', '4', '3', '163', '1499256550', '1499256550', '0', '0', '595cd6e63f375', '27', 'NULL')"; $this->connection->executeQuery($query); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } return new \OC_OCS_Result(); } diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index eea3ada08aad..940bf457c66a 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -35,6 +35,7 @@ namespace OC\Files\Cache; +use OC\Cache\CappedMemoryCache; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; use \OCP\Files\IMimeTypeLoader; @@ -78,6 +79,18 @@ class Cache implements ICache { */ protected $connection; + /** + * caches 2048 filecache rows + * + * FIXME first, refactor this class as an entity and add any necessary methods to its mapper + * FIXME then, get rid of all the places that directly touch the tables + * FIXME finally, make this private non-static + * + * @var CappedMemoryCache + * @deprecated You are not supposed to mess with the filecache directly + */ + public static $metaDataCache = null; + /** * @param \OC\Files\Storage\Storage|string $storage */ @@ -94,6 +107,10 @@ public function __construct($storage) { $this->storageCache = new Storage($storage); $this->mimetypeLoader = \OC::$server->getMimeTypeLoader(); $this->connection = \OC::$server->getDatabaseConnection(); + + if (self::$metaDataCache === null) { + self::$metaDataCache = new CappedMemoryCache(2048); + } } /** @@ -112,7 +129,16 @@ public function getNumericStorageId() { * @return ICacheEntry|false the cache entry as array of false if the file is not found in the cache */ public function get($file) { - if (\is_string($file) or $file == '') { + $key = $this->getNumericStorageId().'-get-'.$file; + if (self::$metaDataCache->hasKey($key)) { // use hasKey to also cache false + $entry = self::$metaDataCache->get($key); + if ($entry instanceof ICacheEntry) { + $entry = clone $entry; // clone to prevent changes affecting our cached clone + } + return $entry; + } + + if (\is_string($file) || $file === '') { // normalize file $file = $this->normalize($file); @@ -128,45 +154,21 @@ public function get($file) { $result = $this->connection->executeQuery($sql, $params); $data = $result->fetch(); - //FIXME hide this HACK in the next database layer, or just use doctrine and get rid of MDB2 and PDO - //PDO returns false, MDB2 returns null, oracle always uses MDB2, so convert null to false - if ($data === null) { - $data = false; - } - - //merge partial data - if($data) { - //fix types - $data['fileid'] = (int)$data['fileid']; - $data['parent'] = (int)$data['parent']; - $data['size'] = 0 + $data['size']; - $data['mtime'] = (int)$data['mtime']; - $data['storage_mtime'] = (int)$data['storage_mtime']; - $data['encryptedVersion'] = (int)$data['encrypted']; - $data['encrypted'] = (bool)$data['encrypted']; - $data['storage'] = $this->storageId; - $data['mimetype'] = $this->mimetypeLoader->getMimetypeById($data['mimetype']); - $data['mimepart'] = $this->mimetypeLoader->getMimetypeById($data['mimepart']); - if ($data['storage_mtime'] == 0) { - $data['storage_mtime'] = $data['mtime']; - } - $data['permissions'] = (int)$data['permissions']; - // Oracle stores empty strings as null... - if (\is_null($data['name'])) { - $data['name'] = ''; - } - if (\is_null($data['path'])) { - $data['path'] = ''; - } - return new CacheEntry($data); - } else if (!$data and \is_string($file)) { - if (isset($this->partial[$file])) { - $data = $this->partial[$file]; - } - return $data; + if ($data) { + $this->fixTypes($data); + $entry = new CacheEntry($data); + self::$metaDataCache->set($key, clone $entry); + $getIdkey = $this->getNumericStorageId().'-getId-'.$data['path']; + self::$metaDataCache->set($getIdkey, 0 + $data['fileid']); + $getPathByIdkey = $this->getNumericStorageId().'-getPathById-'.$data['fileid']; + self::$metaDataCache->set($getPathByIdkey, $data['path']); + } else if (\is_string($file) && isset($this->partial[$file])) { + $entry = $this->partial[$file]; // return, no need cache partial entries } else { - return false; + $entry = false; + self::$metaDataCache->set($key, $entry); } + return $entry; } /** @@ -194,21 +196,48 @@ public function getFolderContentsById($fileId) { $result = $this->connection->executeQuery($sql, [$fileId]); $files = $result->fetchAll(); foreach ($files as &$file) { - $file['mimetype'] = $this->mimetypeLoader->getMimetypeById($file['mimetype']); - $file['mimepart'] = $this->mimetypeLoader->getMimetypeById($file['mimepart']); - if ($file['storage_mtime'] == 0) { - $file['storage_mtime'] = $file['mtime']; - } - $file['permissions'] = (int)$file['permissions']; - $file['mtime'] = (int)$file['mtime']; - $file['storage_mtime'] = (int)$file['storage_mtime']; - $file['size'] = 0 + $file['size']; + $this->fixTypes($file); } return \array_map(function (array $data) { - return new CacheEntry($data); + $entry = new CacheEntry($data); + $getKey = $this->getNumericStorageId().'-get-'.$data['path']; + self::$metaDataCache->set($getKey, clone $entry); + $getKey = $this->getNumericStorageId().'-get-'.$data['fileid']; + self::$metaDataCache->set($getKey, clone $entry); + $getIdkey = $this->getNumericStorageId().'-getId-'.$data['path']; + self::$metaDataCache->set($getIdkey, 0 + $data['fileid']); + $getPathByIdkey = $this->getNumericStorageId().'-getPathById-'.$data['fileid']; + self::$metaDataCache->set($getPathByIdkey, $data['path']); + return $entry; }, $files); - } else { - return []; + } + + return []; + } + + private function fixTypes(array &$data) { + $data['fileid'] = 0 + $data['fileid']; + $data['parent'] = 0 + $data['parent']; + $data['size'] = 0 + $data['size']; + $data['mtime'] = (int)$data['mtime']; + $data['encryptedVersion'] = (int)$data['encrypted']; + $data['encrypted'] = (bool)$data['encrypted']; + $data['storage'] = $this->storageId; + $data['mimetype'] = $this->mimetypeLoader->getMimetypeById($data['mimetype']); + $data['mimepart'] = $this->mimetypeLoader->getMimetypeById($data['mimepart']); + if (isset($data['storage_mtime'])) { + $data['storage_mtime'] = (int)$data['storage_mtime']; + if ($data['storage_mtime'] === 0) { + $data['storage_mtime'] = $data['mtime']; + } + } + $data['permissions'] = (int)$data['permissions']; + // Oracle stores empty strings as null... + if ($data['name'] === null) { + $data['name'] = ''; + } + if ($data['path'] === null) { + $data['path'] = ''; } } @@ -225,9 +254,9 @@ public function put($file, array $data) { if (($id = $this->getId($file)) > -1) { $this->update($id, $data); return $id; - } else { - return $this->insert($file, $data); } + + return $this->insert($file, $data); } /** @@ -240,6 +269,8 @@ public function put($file, array $data) { * @throws \RuntimeException */ public function insert($file, array $data) { + self::$metaDataCache->clear(); + // normalize file $file = $this->normalize($file); @@ -290,7 +321,6 @@ public function insert($file, array $data) { * @param array $data [$key => $value] the metadata to update, only the fields provided in the array will be updated, non-provided values will remain unchanged */ public function update($id, array $data) { - if (isset($data['path'])) { // normalize path $data['path'] = $this->normalize($data['path']); @@ -316,6 +346,7 @@ public function update($id, array $data) { ') AND `fileid` = ? '; $this->connection->executeQuery($sql, $params); + self::$metaDataCache->clear(); } /** @@ -382,16 +413,26 @@ protected function buildParts(array $data) { public function getId($file) { // normalize file $file = $this->normalize($file); + $key = $this->getNumericStorageId().'-getId-'.$file; + $id = self::$metaDataCache->get($key); + if (\is_numeric($id)) { + return $id; + } $pathHash = \md5($file); $sql = 'SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` = ?'; $result = $this->connection->executeQuery($sql, [$this->getNumericStorageId(), $pathHash]); if ($row = $result->fetch()) { - return $row['fileid']; + $id = 0 + $row['fileid']; + $getPathByIdkey = $this->getNumericStorageId().'-getPathById-'.$id; + self::$metaDataCache->set($getPathByIdkey, $file); } else { - return -1; + $id = -1; } + self::$metaDataCache->set($key, $id); + + return $id; } /** @@ -403,10 +444,10 @@ public function getId($file) { public function getParentId($file) { if ($file === '') { return -1; - } else { - $parent = $this->getParentPath($file); - return (int)$this->getId($parent); } + + $parent = $this->getParentPath($file); + return (int)$this->getId($parent); } private function getParentPath($path) { @@ -424,7 +465,7 @@ private function getParentPath($path) { * @return bool */ public function inCache($file) { - return $this->getId($file) != -1; + return $this->getId($file) !== -1; } /** @@ -441,6 +482,7 @@ public function remove($file) { if ($entry['mimetype'] === 'httpd/unix-directory') { $this->removeChildren($entry); } + self::$metaDataCache->clear(); } /** @@ -469,6 +511,7 @@ private function removeChildren($entry) { } $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `parent` = ?'; $this->connection->executeQuery($sql, [$entry['fileid']]); + self::$metaDataCache->clear(); } /** @@ -513,10 +556,10 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { list($sourceStorageId, $sourcePath) = $sourceCache->getMoveInfo($sourcePath); list($targetStorageId, $targetPath) = $this->getMoveInfo($targetPath); - if (\is_null($sourceStorageId) || $sourceStorageId === false) { + if ($sourceStorageId === null|| $sourceStorageId === false) { throw new \Exception('Invalid source storage id: ' . $sourceStorageId); } - if (\is_null($targetStorageId) || $targetStorageId === false) { + if ($targetStorageId === null || $targetStorageId === false) { throw new \Exception('Invalid target storage id: ' . $targetStorageId); } @@ -544,12 +587,14 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { } else { $this->moveFromCacheFallback($sourceCache, $sourcePath, $targetPath); } + self::$metaDataCache->clear(); } /** * remove all entries for files that are stored on the storage from the cache */ public function clear() { + self::$metaDataCache->clear(); Storage::remove($this->storageId); } @@ -575,16 +620,16 @@ public function getStatus($file) { if ($row = $result->fetch()) { if ((int)$row['size'] === -1) { return self::SHALLOW; - } else { - return self::COMPLETE; - } - } else { - if (isset($this->partial[$file])) { - return self::PARTIAL; - } else { - return self::NOT_FOUND; } + + return self::COMPLETE; } + + if (isset($this->partial[$file])) { + return self::PARTIAL; + } + + return self::NOT_FOUND; } /** @@ -609,8 +654,7 @@ public function search($pattern) { $files = []; while ($row = $result->fetch()) { - $row['mimetype'] = $this->mimetypeLoader->getMimetypeById($row['mimetype']); - $row['mimepart'] = $this->mimetypeLoader->getMimetypeById($row['mimepart']); + $this->fixTypes($row); $files[] = $row; } return \array_map(function(array $data) { @@ -637,8 +681,7 @@ public function searchByMime($mimetype) { $result = $this->connection->executeQuery($sql, [$mimetype, $this->getNumericStorageId()]); $files = []; while ($row = $result->fetch()) { - $row['mimetype'] = $this->mimetypeLoader->getMimetypeById($row['mimetype']); - $row['mimepart'] = $this->mimetypeLoader->getMimetypeById($row['mimepart']); + $this->fixTypes($row); $files[] = $row; } return \array_map(function (array $data) { @@ -686,6 +729,7 @@ public function searchByTag($tag, $userId) { ); $files = []; while ($row = $result->fetch()) { + $this->fixTypes($row); $files[] = $row; } return \array_map(function (array $data) { @@ -719,7 +763,7 @@ public function correctFolderSize($path, $data = null) { */ public function calculateFolderSize($path, $entry = null) { $totalSize = 0; - if (\is_null($entry) or !isset($entry['fileid'])) { + if ($entry === null || !isset($entry['fileid'])) { $entry = $this->get($path); } if (isset($entry['mimetype']) && $entry['mimetype'] === 'httpd/unix-directory') { @@ -782,9 +826,9 @@ public function getIncomplete() { $query->execute([$this->getNumericStorageId()]); if ($row = $query->fetch()) { return $row['path']; - } else { - return false; } + + return false; } /** @@ -794,17 +838,28 @@ public function getIncomplete() { * @return string|null the path of the file (relative to the storage) or null if a file with the given id does not exists within this cache */ public function getPathById($id) { + $key = $this->getNumericStorageId().'-getPathById-'.$id; + if (self::$metaDataCache->hasKey($key)) { // use hasKey to also cache null + return self::$metaDataCache->get($key); + } + $sql = 'SELECT `path` FROM `*PREFIX*filecache` WHERE `fileid` = ? AND `storage` = ?'; $result = $this->connection->executeQuery($sql, [$id, $this->getNumericStorageId()]); if ($row = $result->fetch()) { // Oracle stores empty strings as null... if ($row['path'] === null) { - return ''; + $path = ''; + } else { + $path = $row['path']; } - return $row['path']; + $getPathByIdkey = $this->getNumericStorageId().'-getId-'.$path; + self::$metaDataCache->set($getPathByIdkey, 0 + $id); } else { - return null; + $path = null; } + self::$metaDataCache->set($key, $path); + + return $path; } /** @@ -829,9 +884,9 @@ static public function getById($id) { if ($id = Storage::getStorageId($numericId)) { return [$id, $path]; - } else { - return null; } + + return null; } /** @@ -841,7 +896,6 @@ static public function getById($id) { * @return string */ public function normalize($path) { - return \trim(\OC_Util::normalizeUnicode($path), '/'); } } diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php index cf33048e4856..587ba7cec252 100644 --- a/lib/private/Files/Cache/Propagator.php +++ b/lib/private/Files/Cache/Propagator.php @@ -96,6 +96,10 @@ public function propagateChange($internalPath, $time, $sizeDifference = 0) { } $builder->execute(); + + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } protected function getParents($path) { @@ -180,6 +184,10 @@ public function commitBatch() { $this->batch = []; $this->connection->commit(); + + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } } diff --git a/lib/private/Files/Cache/Storage.php b/lib/private/Files/Cache/Storage.php index 768f38a808f0..af6840cb335a 100644 --- a/lib/private/Files/Cache/Storage.php +++ b/lib/private/Files/Cache/Storage.php @@ -270,6 +270,9 @@ public static function remove($storageId) { if (!\is_null($numericId)) { $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `storage` = ?'; \OC_DB::executeAudited($sql, [$numericId]); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } } } diff --git a/lib/private/Files/Type/Loader.php b/lib/private/Files/Type/Loader.php index fd6b52bbbe2f..af9c6a0d1a38 100644 --- a/lib/private/Files/Type/Loader.php +++ b/lib/private/Files/Type/Loader.php @@ -21,6 +21,7 @@ namespace OC\Files\Type; +use OC\Files\Cache\Cache; use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; @@ -171,6 +172,9 @@ public function updateFilecache($ext, $mimetypeId) { ->andWhere($update->expr()->like( $update->createFunction('LOWER(`name`)'), $update->createNamedParameter($ext) )); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } return $update->execute(); } diff --git a/lib/private/Repair/FillETags.php b/lib/private/Repair/FillETags.php index d0e2ab01384d..aa61e15d9162 100644 --- a/lib/private/Repair/FillETags.php +++ b/lib/private/Repair/FillETags.php @@ -23,6 +23,7 @@ namespace OC\Repair; +use OC\Files\Cache\Cache; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; @@ -50,6 +51,9 @@ public function run(IOutput $output) { ->orWhere($qb->expr()->isNull('etag')); $result = $qb->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $output->info("ETags have been fixed for $result files/folders."); } } diff --git a/lib/private/Repair/RepairMimeTypes.php b/lib/private/Repair/RepairMimeTypes.php index 08b17f5b4ee5..6634cf3a15c1 100644 --- a/lib/private/Repair/RepairMimeTypes.php +++ b/lib/private/Repair/RepairMimeTypes.php @@ -29,6 +29,7 @@ namespace OC\Repair; +use OC\Files\Cache\Cache; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; @@ -127,6 +128,9 @@ private function repairMimetypes($wrongMimetypes) { // delete wrong mimetype \OC_DB::executeAudited(self::deleteStmt(), [$wrongId]); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } } } @@ -134,6 +138,9 @@ private function repairMimetypes($wrongMimetypes) { private function updateMimetypes($updatedMimetypes) { if (empty($this->folderMimeTypeId)) { $result = \OC_DB::executeAudited(self::getIdStmt(), ['httpd/unix-directory']); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $this->folderMimeTypeId = (int)$result->fetchOne(); } @@ -152,6 +159,9 @@ private function updateMimetypes($updatedMimetypes) { // change mimetype for files with x extension \OC_DB::executeAudited(self::updateByNameStmt(), [$mimetypeId, $this->folderMimeTypeId, $mimetypeId, '%.' . $extension]); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } } diff --git a/tests/lib/Files/Cache/ScannerTest.php b/tests/lib/Files/Cache/ScannerTest.php index 0ad839275c9f..fae8ca42dd90 100644 --- a/tests/lib/Files/Cache/ScannerTest.php +++ b/tests/lib/Files/Cache/ScannerTest.php @@ -8,6 +8,7 @@ namespace Test\Files\Cache; +use OC\Files\Cache\Cache; use OC\Files\Cache\CacheEntry; use OC\Files\Storage\Storage; use OCA\Files_Sharing\ISharedStorage; @@ -286,6 +287,9 @@ public function testRepairParent() { // delete the folder without removing the childs $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `fileid` = ?'; \OC_DB::executeAudited($sql, [$oldFolderId]); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $cachedData = $this->cache->get('folder/bar.txt'); $this->assertEquals($oldFolderId, $cachedData['parent']); @@ -310,6 +314,9 @@ public function testRepairParentShallow() { // delete the folder without removing the childs $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `fileid` = ?'; \OC_DB::executeAudited($sql, [$oldFolderId]); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $cachedData = $this->cache->get('folder/bar.txt'); $this->assertEquals($oldFolderId, $cachedData['parent']); diff --git a/tests/lib/Files/Config/UserMountCacheTest.php b/tests/lib/Files/Config/UserMountCacheTest.php index e66e6dc92c81..d48110f8b0b7 100644 --- a/tests/lib/Files/Config/UserMountCacheTest.php +++ b/tests/lib/Files/Config/UserMountCacheTest.php @@ -9,6 +9,7 @@ namespace Test\Files\Config; use OC\DB\QueryBuilder\Literal; +use OC\Files\Cache\Cache; use OC\Files\Config\UserMountCache; use OC\Files\Mount\MountPoint; use OC\Log; @@ -101,6 +102,9 @@ public function tearDown() { ->where($builder->expr()->eq('fileid', new Literal($fileId))) ->execute(); } + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } private function getStorage($storageId, $rootId) { @@ -330,6 +334,9 @@ private function createCacheEntry($internalPath, $storageId) { 'etag' => '', 'permissions' => 31 ], ['storage', 'path_hash']); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $id = (int)$this->connection->lastInsertId('*PREFIX*filecache'); $this->fileIds[] = $id; return $id; diff --git a/tests/lib/Repair/CleanTagsTest.php b/tests/lib/Repair/CleanTagsTest.php index bfba40e96c8f..5a3f28575ea7 100644 --- a/tests/lib/Repair/CleanTagsTest.php +++ b/tests/lib/Repair/CleanTagsTest.php @@ -7,6 +7,7 @@ */ namespace Test\Repair; +use OC\Files\Cache\Cache; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Migration\IOutput; @@ -66,6 +67,9 @@ protected function cleanUpTables() { $qb->delete('filecache') ->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } public function testRun() { @@ -186,6 +190,9 @@ protected function getFileID() { 'path_hash' => $qb->createNamedParameter(\md5($fileName)), ]) ->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } $this->createdFile = (int) $this->getLastInsertID('filecache', 'fileid'); return $this->createdFile; diff --git a/tests/lib/Share20/DefaultShareProviderTest.php b/tests/lib/Share20/DefaultShareProviderTest.php index 327a98697127..be6deea1bb69 100644 --- a/tests/lib/Share20/DefaultShareProviderTest.php +++ b/tests/lib/Share20/DefaultShareProviderTest.php @@ -22,6 +22,7 @@ use OC\Authentication\Token\DefaultTokenMapper; use OC\Share20\DefaultShareProvider; +use OC\Files\Cache\Cache; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\File; use OCP\Files\Folder; @@ -79,6 +80,9 @@ public function setUp() { public function tearDown() { $this->dbConn->getQueryBuilder()->delete('share')->execute(); $this->dbConn->getQueryBuilder()->delete('filecache')->execute(); + if (isset(Cache::$metaDataCache)) { + Cache::$metaDataCache->clear(); + } $this->dbConn->getQueryBuilder()->delete('storages')->execute(); } @@ -815,6 +819,9 @@ private function createTestFileEntry($path, $storage = 1) { 'name' => $qb->expr()->literal(\basename($path)), ]); $this->assertEquals(1, $qb->execute()); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } return $qb->getLastInsertId(); } diff --git a/tests/lib/TestCase.php b/tests/lib/TestCase.php index d3763170c461..812c79e45370 100644 --- a/tests/lib/TestCase.php +++ b/tests/lib/TestCase.php @@ -25,6 +25,7 @@ use DOMDocument; use DOMNode; use OC\Command\QueueBus; +use OC\Files\Cache\Cache; use OC\Files\Filesystem; use OC\Template\Base; use OC_Defaults; @@ -292,6 +293,9 @@ static protected function tearDownAfterClassCleanStorages(IQueryBuilder $queryBu static protected function tearDownAfterClassCleanFileCache(IQueryBuilder $queryBuilder) { $queryBuilder->delete('filecache') ->execute(); + if (Cache::$metaDataCache !== null) { + Cache::$metaDataCache->clear(); + } } /**