From c5803ef0db6f781a7e9322e4f52fcc96d594412e Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 15 Jun 2017 13:59:06 +0200 Subject: [PATCH 01/13] fix moving folders out of a cache jail Signed-off-by: Robin Appelman --- lib/private/Files/Cache/Wrapper/CacheJail.php | 12 ++++- .../lib/Files/Cache/Wrapper/CacheJailTest.php | 50 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/lib/private/Files/Cache/Wrapper/CacheJail.php b/lib/private/Files/Cache/Wrapper/CacheJail.php index d8bdca6a3c451..a275d75ea3220 100644 --- a/lib/private/Files/Cache/Wrapper/CacheJail.php +++ b/lib/private/Files/Cache/Wrapper/CacheJail.php @@ -93,7 +93,7 @@ protected function filterCacheEntry($entry) { * get the stored metadata of a file or folder * * @param string /int $file - * @return array|false + * @return ICacheEntry|false */ public function get($file) { if (is_string($file) or $file == '') { @@ -178,6 +178,16 @@ public function move($source, $target) { $this->getCache()->move($this->getSourcePath($source), $this->getSourcePath($target)); } + /** + * Get the storage id and path needed for a move + * + * @param string $path + * @return array [$storageId, $internalPath] + */ + protected function getMoveInfo($path) { + return [$this->getNumericStorageId(), $this->getSourcePath($path)]; + } + /** * remove all entries for files that are stored on the storage from the cache */ diff --git a/tests/lib/Files/Cache/Wrapper/CacheJailTest.php b/tests/lib/Files/Cache/Wrapper/CacheJailTest.php index e3043c50d5768..f26e3a59f1c28 100644 --- a/tests/lib/Files/Cache/Wrapper/CacheJailTest.php +++ b/tests/lib/Files/Cache/Wrapper/CacheJailTest.php @@ -8,6 +8,7 @@ namespace Test\Files\Cache\Wrapper; +use OC\Files\Cache\Wrapper\CacheJail; use Test\Files\Cache\CacheTest; /** @@ -80,4 +81,53 @@ function testGetIncomplete() { //not supported $this->assertTrue(true); } + + function testMoveFromJail() { + $folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->sourceCache->put('source', $folderData); + $this->sourceCache->put('source/foo', $folderData); + $this->sourceCache->put('source/foo/bar', $folderData); + $this->sourceCache->put('target', $folderData); + + $jail = new CacheJail($this->sourceCache, 'source'); + + $this->sourceCache->moveFromCache($jail, 'foo', 'target/foo'); + + $this->assertTrue($this->sourceCache->inCache('target/foo')); + $this->assertTrue($this->sourceCache->inCache('target/foo/bar')); + } + + function testMoveToJail() { + $folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->sourceCache->put('source', $folderData); + $this->sourceCache->put('source/foo', $folderData); + $this->sourceCache->put('source/foo/bar', $folderData); + $this->sourceCache->put('target', $folderData); + + $jail = new CacheJail($this->sourceCache, 'target'); + + $jail->moveFromCache($this->sourceCache, 'source/foo', 'foo'); + + $this->assertTrue($this->sourceCache->inCache('target/foo')); + $this->assertTrue($this->sourceCache->inCache('target/foo/bar')); + } + + function testMoveBetweenJail() { + $folderData = array('size' => 100, 'mtime' => 50, 'mimetype' => 'httpd/unix-directory'); + + $this->sourceCache->put('source', $folderData); + $this->sourceCache->put('source/foo', $folderData); + $this->sourceCache->put('source/foo/bar', $folderData); + $this->sourceCache->put('target', $folderData); + + $jail = new CacheJail($this->sourceCache, 'target'); + $sourceJail = new CacheJail($this->sourceCache, 'source'); + + $jail->moveFromCache($sourceJail, 'foo', 'foo'); + + $this->assertTrue($this->sourceCache->inCache('target/foo')); + $this->assertTrue($this->sourceCache->inCache('target/foo/bar')); + } } From 0597d3f72f6941c2d0343c70dbf1a023fde7d00d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 20 Jun 2017 16:31:52 +0200 Subject: [PATCH 02/13] Add repair step for invalid paths Signed-off-by: Robin Appelman --- lib/private/Repair.php | 4 +- lib/private/Repair/RepairInvalidPaths.php | 121 ++++++++++++++++++++ tests/lib/Repair/RepairInvalidPathsTest.php | 111 ++++++++++++++++++ 3 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 lib/private/Repair/RepairInvalidPaths.php create mode 100644 tests/lib/Repair/RepairInvalidPathsTest.php diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 16236fd6bcc72..79925d872e0ae 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -43,6 +43,7 @@ use OC\Repair\RemoveGetETagEntries; use OC\Repair\RemoveOldShares; use OC\Repair\RemoveRootShares; +use OC\Repair\RepairInvalidPaths; use OC\Repair\SharePropagation; use OC\Repair\SqliteAutoincrement; use OC\Repair\DropOldTables; @@ -163,6 +164,7 @@ public static function getRepairSteps() { \OC::$server->getConfig() ), new FixMountStorages(\OC::$server->getDatabaseConnection()), + new RepairInvalidPaths(\OC::$server->getDatabaseConnection()) ]; } @@ -174,7 +176,7 @@ public static function getRepairSteps() { */ public static function getExpensiveRepairSteps() { return [ - new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()), + new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()) ]; } diff --git a/lib/private/Repair/RepairInvalidPaths.php b/lib/private/Repair/RepairInvalidPaths.php new file mode 100644 index 0000000000000..cdd0906295fa4 --- /dev/null +++ b/lib/private/Repair/RepairInvalidPaths.php @@ -0,0 +1,121 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Repair; + + +use OCP\IDBConnection; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class RepairInvalidPaths implements IRepairStep { + /** @var IDBConnection */ + private $connection; + + public function __construct(IDBConnection $connection) { + $this->connection = $connection; + } + + + public function getName() { + return 'Repair invalid paths in file cache'; + } + + private function getInvalidEntries() { + $builder = $this->connection->getQueryBuilder(); + + $computedPath = $builder->func()->concat( + 'p.path', + $builder->func()->concat($builder->createNamedParameter('/'), 'f.name') + ); + + //select f.path, f.parent,p.path from oc_filecache f inner join oc_filecache p on f.parent=p.fileid and p.path!='' where f.path != p.path || '/' || f.name; + $query = $builder->select('f.fileid', 'f.path', 'p.path AS parent_path', 'f.name', 'f.parent', 'f.storage') + ->from('filecache', 'f') + ->innerJoin('f', 'filecache', 'p', $builder->expr()->andX( + $builder->expr()->eq('f.parent', 'p.fileid'), + $builder->expr()->neq('p.name', $builder->createNamedParameter('')) + )) + ->where($builder->expr()->neq('f.path', $computedPath)); + + return $query->execute()->fetchAll(); + } + + private function getId($storage, $path) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->select('fileid') + ->from('filecache') + ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storage))) + ->andWhere($builder->expr()->eq('path', $builder->createNamedParameter($path))); + + return $query->execute()->fetchColumn(); + } + + private function update($fileid, $newPath) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->update('filecache') + ->set('path', $builder->createNamedParameter($newPath)) + ->set('path_hash', $builder->createNamedParameter(md5($newPath))) + ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileid))); + + $query->execute(); + } + + private function reparent($from, $to) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->update('filecache') + ->set('parent', $builder->createNamedParameter($to)) + ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($from))); + $query->execute(); + } + + private function delete($fileid) { + $builder = $this->connection->getQueryBuilder(); + + $query = $builder->delete('filecache') + ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileid))); + $query->execute(); + } + + private function repair() { + $entries = $this->getInvalidEntries(); + foreach ($entries as $entry) { + $calculatedPath = $entry['parent_path'] . '/' . $entry['name']; + if ($newId = $this->getId($entry['storage'], $calculatedPath)) { + // a new entry with the correct path has already been created, reuse that one and delete the incorrect entry + $this->reparent($entry['fileid'], $newId); + $this->delete($entry['fileid']); + } else { + $this->update($entry['fileid'], $calculatedPath); + } + } + return count($entries); + } + + public function run(IOutput $output) { + $count = $this->repair(); + + $output->info('Repaired ' . $count . ' paths'); + } +} diff --git a/tests/lib/Repair/RepairInvalidPathsTest.php b/tests/lib/Repair/RepairInvalidPathsTest.php new file mode 100644 index 0000000000000..c6c1bbb19bc9e --- /dev/null +++ b/tests/lib/Repair/RepairInvalidPathsTest.php @@ -0,0 +1,111 @@ + + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace Test\Repair; + +use OC\Files\Cache\Cache; +use OC\Files\Storage\Temporary; +use OC\Repair\RepairInvalidPaths; +use OCP\Migration\IOutput; +use Test\TestCase; + +/** + * @group DB + */ +class RepairInvalidPathsTest extends TestCase { + /** @var Temporary */ + private $storage; + /** @var Cache */ + private $cache; + /** @var RepairInvalidPaths */ + private $repair; + + protected function setUp() { + parent::setUp(); + + $this->storage = new Temporary(); + $this->cache = $this->storage->getCache(); + $this->repair = new RepairInvalidPaths(\OC::$server->getDatabaseConnection()); + } + + protected function tearDown() { + $this->cache->clear(); + + return parent::tearDown(); + } + + public function testRepairNonDuplicate() { + $this->storage->mkdir('foo/bar/asd'); + $this->storage->mkdir('foo2'); + $this->storage->getScanner()->scan(''); + + $folderId = $this->cache->getId('foo/bar'); + $newParentFolderId = $this->cache->getId('foo2'); + // failed rename, moved entry is updated but not it's children + $this->cache->update($folderId, ['path' => 'foo2/bar', 'parent' => $newParentFolderId]); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo/bar/asd')); + $this->assertFalse($this->cache->inCache('foo2/bar/asd')); + + $this->assertEquals($folderId, $this->cache->get('foo/bar/asd')['parent']); + + $this->repair->run($this->createMock(IOutput::class)); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo2/bar/asd')); + $this->assertFalse($this->cache->inCache('foo/bar/asd')); + + $this->assertEquals($folderId, $this->cache->get('foo2/bar/asd')['parent']); + $this->assertEquals($folderId, $this->cache->getId('foo2/bar')); + } + + public function testRepairDuplicate() { + $this->storage->mkdir('foo/bar/asd'); + $this->storage->mkdir('foo2'); + $this->storage->getScanner()->scan(''); + + $folderId = $this->cache->getId('foo/bar'); + $newParentFolderId = $this->cache->getId('foo2'); + // failed rename, moved entry is updated but not it's children + $this->cache->update($folderId, ['path' => 'foo2/bar', 'parent' => $newParentFolderId]); + $this->storage->rename('foo/bar', 'foo2/bar'); + $this->storage->mkdir('foo2/bar/asd/foo'); + + // usage causes the renamed subfolder to be scanned + $this->storage->getScanner()->scan('foo2/bar/asd'); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo/bar/asd')); + $this->assertTrue($this->cache->inCache('foo2/bar/asd')); + + $this->assertEquals($folderId, $this->cache->get('foo/bar/asd')['parent']); + + $this->repair->run($this->createMock(IOutput::class)); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo2/bar/asd')); + $this->assertFalse($this->cache->inCache('foo/bar/asd')); + + $this->assertEquals($this->cache->getId('foo2/bar'), $this->cache->get('foo2/bar/asd')['parent']); + $this->assertEquals($this->cache->getId('foo2/bar/asd'), $this->cache->get('foo2/bar/asd/foo')['parent']); + } +} From daa2c869659f282a19e11503a4d7a2a97bbce1e8 Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Wed, 21 Jun 2017 16:50:20 -0500 Subject: [PATCH 03/13] Run repair step only once Signed-off-by: Morris Jobke --- lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + lib/private/Repair.php | 2 +- lib/private/Repair/{ => NC13}/RepairInvalidPaths.php | 10 +++++++--- version.php | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) rename lib/private/Repair/{ => NC13}/RepairInvalidPaths.php (91%) diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 19f995e3b6f02..447b062dc05df 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -683,6 +683,7 @@ 'OC\\Repair\\NC11\\FixMountStorages' => $baseDir . '/lib/private/Repair/NC11/FixMountStorages.php', 'OC\\Repair\\NC11\\MoveAvatars' => $baseDir . '/lib/private/Repair/NC11/MoveAvatars.php', 'OC\\Repair\\NC11\\MoveAvatarsBackgroundJob' => $baseDir . '/lib/private/Repair/NC11/MoveAvatarsBackgroundJob.php', + 'OC\\Repair\\NC13\\RepairInvalidPaths' => $baseDir . '/lib/private/Repair/NC13/RepairInvalidPaths.php', 'OC\\Repair\\OldGroupMembershipShares' => $baseDir . '/lib/private/Repair/OldGroupMembershipShares.php', 'OC\\Repair\\Preview' => $baseDir . '/lib/private/Repair/Preview.php', 'OC\\Repair\\RemoveGetETagEntries' => $baseDir . '/lib/private/Repair/RemoveGetETagEntries.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index d5fe21da19f8a..019f733e30741 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -713,6 +713,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Repair\\NC11\\FixMountStorages' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/FixMountStorages.php', 'OC\\Repair\\NC11\\MoveAvatars' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/MoveAvatars.php', 'OC\\Repair\\NC11\\MoveAvatarsBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/MoveAvatarsBackgroundJob.php', + 'OC\\Repair\\NC13\\RepairInvalidPaths' => __DIR__ . '/../../..' . '/lib/private/Repair/NC13/RepairInvalidPaths.php', 'OC\\Repair\\OldGroupMembershipShares' => __DIR__ . '/../../..' . '/lib/private/Repair/OldGroupMembershipShares.php', 'OC\\Repair\\Preview' => __DIR__ . '/../../..' . '/lib/private/Repair/Preview.php', 'OC\\Repair\\RemoveGetETagEntries' => __DIR__ . '/../../..' . '/lib/private/Repair/RemoveGetETagEntries.php', diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 79925d872e0ae..5b030e0b8a7bd 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -43,7 +43,7 @@ use OC\Repair\RemoveGetETagEntries; use OC\Repair\RemoveOldShares; use OC\Repair\RemoveRootShares; -use OC\Repair\RepairInvalidPaths; +use OC\Repair\NC13\RepairInvalidPaths; use OC\Repair\SharePropagation; use OC\Repair\SqliteAutoincrement; use OC\Repair\DropOldTables; diff --git a/lib/private/Repair/RepairInvalidPaths.php b/lib/private/Repair/NC13/RepairInvalidPaths.php similarity index 91% rename from lib/private/Repair/RepairInvalidPaths.php rename to lib/private/Repair/NC13/RepairInvalidPaths.php index cdd0906295fa4..8551f8261e217 100644 --- a/lib/private/Repair/RepairInvalidPaths.php +++ b/lib/private/Repair/NC13/RepairInvalidPaths.php @@ -19,7 +19,7 @@ * */ -namespace OC\Repair; +namespace OC\Repair\NC13; use OCP\IDBConnection; @@ -114,8 +114,12 @@ private function repair() { } public function run(IOutput $output) { - $count = $this->repair(); + $versionFromBeforeUpdate = $this->config->getSystemValue('version', '0.0.0'); + // was added to 12.0.0.30 and 13.0.0.1 + if (version_compare($versionFromBeforeUpdate, '12.0.0.30', '<') || version_compare($versionFromBeforeUpdate, '13.0.0.0', '==')) { + $count = $this->repair(); - $output->info('Repaired ' . $count . ' paths'); + $output->info('Repaired ' . $count . ' paths'); + } } } diff --git a/version.php b/version.php index 7ab4358496054..ec9fccb30709c 100644 --- a/version.php +++ b/version.php @@ -26,7 +26,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patchlevel // when updating major/minor version number. -$OC_Version = array(11, 0, 5, 1); +$OC_Version = array(11, 0, 5, 2); // The human readable string $OC_VersionString = '11.0.5'; From f01bd935edc92b8838f67f3db7edd3d0a0d32188 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 29 Jun 2017 14:45:08 +0200 Subject: [PATCH 04/13] adjust to moved repair step Signed-off-by: Robin Appelman --- lib/private/Repair.php | 2 +- lib/private/Repair/NC13/RepairInvalidPaths.php | 6 +++++- tests/lib/Repair/RepairInvalidPathsTest.php | 10 ++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 5b030e0b8a7bd..320f39d9a6d8d 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -164,7 +164,7 @@ public static function getRepairSteps() { \OC::$server->getConfig() ), new FixMountStorages(\OC::$server->getDatabaseConnection()), - new RepairInvalidPaths(\OC::$server->getDatabaseConnection()) + new RepairInvalidPaths(\OC::$server->getDatabaseConnection(), \OC::$server->getConfig()) ]; } diff --git a/lib/private/Repair/NC13/RepairInvalidPaths.php b/lib/private/Repair/NC13/RepairInvalidPaths.php index 8551f8261e217..076fbb735c8ee 100644 --- a/lib/private/Repair/NC13/RepairInvalidPaths.php +++ b/lib/private/Repair/NC13/RepairInvalidPaths.php @@ -22,6 +22,7 @@ namespace OC\Repair\NC13; +use OCP\IConfig; use OCP\IDBConnection; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; @@ -29,9 +30,12 @@ class RepairInvalidPaths implements IRepairStep { /** @var IDBConnection */ private $connection; + /** @var IConfig */ + private $config; - public function __construct(IDBConnection $connection) { + public function __construct(IDBConnection $connection, IConfig $config) { $this->connection = $connection; + $this->config = $config; } diff --git a/tests/lib/Repair/RepairInvalidPathsTest.php b/tests/lib/Repair/RepairInvalidPathsTest.php index c6c1bbb19bc9e..b18758585c19b 100644 --- a/tests/lib/Repair/RepairInvalidPathsTest.php +++ b/tests/lib/Repair/RepairInvalidPathsTest.php @@ -23,7 +23,8 @@ use OC\Files\Cache\Cache; use OC\Files\Storage\Temporary; -use OC\Repair\RepairInvalidPaths; +use OC\Repair\NC13\RepairInvalidPaths; +use OCP\IConfig; use OCP\Migration\IOutput; use Test\TestCase; @@ -43,7 +44,12 @@ protected function setUp() { $this->storage = new Temporary(); $this->cache = $this->storage->getCache(); - $this->repair = new RepairInvalidPaths(\OC::$server->getDatabaseConnection()); + $config = $this->createMock(IConfig::class); + $config->expects($this->any()) + ->method('getSystemValue') + ->with('version', '0.0.0') + ->willReturn('12.0.0.0'); + $this->repair = new RepairInvalidPaths(\OC::$server->getDatabaseConnection(), $config); } protected function tearDown() { From 19ce46ed34c7e8b5ec86a8602146a29e4fa261f9 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 12 Jul 2017 13:37:42 +0200 Subject: [PATCH 05/13] use a generator instead of fetching all rows at once Signed-off-by: Robin Appelman --- lib/private/Repair/NC13/RepairInvalidPaths.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/private/Repair/NC13/RepairInvalidPaths.php b/lib/private/Repair/NC13/RepairInvalidPaths.php index 076fbb735c8ee..47a007baf5ffd 100644 --- a/lib/private/Repair/NC13/RepairInvalidPaths.php +++ b/lib/private/Repair/NC13/RepairInvalidPaths.php @@ -60,7 +60,10 @@ private function getInvalidEntries() { )) ->where($builder->expr()->neq('f.path', $computedPath)); - return $query->execute()->fetchAll(); + $result = $query->execute(); + while ($row = $result->fetch()) { + yield $row; + } } private function getId($storage, $path) { @@ -103,8 +106,11 @@ private function delete($fileid) { } private function repair() { + $this->connection->beginTransaction(); $entries = $this->getInvalidEntries(); + $count = 0; foreach ($entries as $entry) { + $count++; $calculatedPath = $entry['parent_path'] . '/' . $entry['name']; if ($newId = $this->getId($entry['storage'], $calculatedPath)) { // a new entry with the correct path has already been created, reuse that one and delete the incorrect entry @@ -114,7 +120,8 @@ private function repair() { $this->update($entry['fileid'], $calculatedPath); } } - return count($entries); + $this->connection->commit(); + return $count; } public function run(IOutput $output) { From a96dd73f6c032c3f50679968d3de1ebce6c6019e Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 12 Jul 2017 15:49:36 +0200 Subject: [PATCH 06/13] chunk getting invalid paths and reuse queries Signed-off-by: Robin Appelman --- .../Repair/NC13/RepairInvalidPaths.php | 88 +++++++++++++------ tests/lib/Repair/RepairInvalidPathsTest.php | 38 ++++++++ 2 files changed, 99 insertions(+), 27 deletions(-) diff --git a/lib/private/Repair/NC13/RepairInvalidPaths.php b/lib/private/Repair/NC13/RepairInvalidPaths.php index 47a007baf5ffd..cf0b9e7783ef7 100644 --- a/lib/private/Repair/NC13/RepairInvalidPaths.php +++ b/lib/private/Repair/NC13/RepairInvalidPaths.php @@ -22,17 +22,25 @@ namespace OC\Repair\NC13; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\IConfig; use OCP\IDBConnection; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; class RepairInvalidPaths implements IRepairStep { + const MAX_ROWS = 1000; + /** @var IDBConnection */ private $connection; /** @var IConfig */ private $config; + private $getIdQuery; + private $updateQuery; + private $reparentQuery; + private $deleteQuery; + public function __construct(IDBConnection $connection, IConfig $config) { $this->connection = $connection; $this->config = $config; @@ -58,51 +66,77 @@ private function getInvalidEntries() { $builder->expr()->eq('f.parent', 'p.fileid'), $builder->expr()->neq('p.name', $builder->createNamedParameter('')) )) - ->where($builder->expr()->neq('f.path', $computedPath)); - - $result = $query->execute(); - while ($row = $result->fetch()) { - yield $row; - } + ->where($builder->expr()->neq('f.path', $computedPath)) + ->setMaxResults(self::MAX_ROWS); + + do { + $result = $query->execute(); + $rows = $result->fetchAll(); + foreach ($rows as $row) { + yield $row; + } + $result->closeCursor(); + } while (count($rows) >= self::MAX_ROWS); } private function getId($storage, $path) { - $builder = $this->connection->getQueryBuilder(); + if (!$this->getIdQuery) { + $builder = $this->connection->getQueryBuilder(); + + $this->getIdQuery = $builder->select('fileid') + ->from('filecache') + ->where($builder->expr()->eq('storage', $builder->createParameter('storage'))) + ->andWhere($builder->expr()->eq('path', $builder->createParameter('path'))); + } - $query = $builder->select('fileid') - ->from('filecache') - ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storage))) - ->andWhere($builder->expr()->eq('path', $builder->createNamedParameter($path))); + $this->getIdQuery->setParameter('storage', $storage, IQueryBuilder::PARAM_INT); + $this->getIdQuery->setParameter('path', $path); - return $query->execute()->fetchColumn(); + return $this->getIdQuery->execute()->fetchColumn(); } private function update($fileid, $newPath) { - $builder = $this->connection->getQueryBuilder(); + if (!$this->updateQuery) { + $builder = $this->connection->getQueryBuilder(); - $query = $builder->update('filecache') - ->set('path', $builder->createNamedParameter($newPath)) - ->set('path_hash', $builder->createNamedParameter(md5($newPath))) - ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileid))); + $this->updateQuery = $builder->update('filecache') + ->set('path', $builder->createParameter('newpath')) + ->set('path_hash', $builder->func()->md5($builder->createParameter('newpath'))) + ->where($builder->expr()->eq('fileid', $builder->createParameter('fileid'))); + } - $query->execute(); + $this->updateQuery->setParameter('newpath', $newPath); + $this->updateQuery->setParameter('fileid', $fileid, IQueryBuilder::PARAM_INT); + + $this->updateQuery->execute(); } private function reparent($from, $to) { - $builder = $this->connection->getQueryBuilder(); + if (!$this->reparentQuery) { + $builder = $this->connection->getQueryBuilder(); + + $this->reparentQuery = $builder->update('filecache') + ->set('parent', $builder->createParameter('to')) + ->where($builder->expr()->eq('fileid', $builder->createParameter('from'))); + } + + $this->reparentQuery->setParameter('from', $from); + $this->reparentQuery->setParameter('to', $to); - $query = $builder->update('filecache') - ->set('parent', $builder->createNamedParameter($to)) - ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($from))); - $query->execute(); + $this->reparentQuery->execute(); } private function delete($fileid) { - $builder = $this->connection->getQueryBuilder(); + if (!$this->deleteQuery) { + $builder = $this->connection->getQueryBuilder(); + + $this->deleteQuery = $builder->delete('filecache') + ->where($builder->expr()->eq('fileid', $builder->createParameter('fileid'))); + } + + $this->deleteQuery->setParameter('fileid', $fileid, IQueryBuilder::PARAM_INT); - $query = $builder->delete('filecache') - ->where($builder->expr()->eq('fileid', $builder->createNamedParameter($fileid))); - $query->execute(); + $this->deleteQuery->execute(); } private function repair() { diff --git a/tests/lib/Repair/RepairInvalidPathsTest.php b/tests/lib/Repair/RepairInvalidPathsTest.php index b18758585c19b..fe848b620732f 100644 --- a/tests/lib/Repair/RepairInvalidPathsTest.php +++ b/tests/lib/Repair/RepairInvalidPathsTest.php @@ -114,4 +114,42 @@ public function testRepairDuplicate() { $this->assertEquals($this->cache->getId('foo2/bar'), $this->cache->get('foo2/bar/asd')['parent']); $this->assertEquals($this->cache->getId('foo2/bar/asd'), $this->cache->get('foo2/bar/asd/foo')['parent']); } + + public function testRepairMultipleNonDuplicate() { + $this->storage->mkdir('foo/bar/asd'); + $this->storage->mkdir('foo/bar2/asd'); + $this->storage->mkdir('foo2'); + $this->storage->getScanner()->scan(''); + + $folderId1 = $this->cache->getId('foo/bar'); + $folderId2 = $this->cache->getId('foo/bar2'); + $newParentFolderId = $this->cache->getId('foo2'); + // failed rename, moved entry is updated but not it's children + $this->cache->update($folderId1, ['path' => 'foo2/bar', 'parent' => $newParentFolderId]); + $this->cache->update($folderId2, ['path' => 'foo2/bar2', 'parent' => $newParentFolderId]); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo2/bar2')); + $this->assertTrue($this->cache->inCache('foo/bar/asd')); + $this->assertTrue($this->cache->inCache('foo/bar2/asd')); + $this->assertFalse($this->cache->inCache('foo2/bar/asd')); + $this->assertFalse($this->cache->inCache('foo2/bar2/asd')); + + $this->assertEquals($folderId1, $this->cache->get('foo/bar/asd')['parent']); + $this->assertEquals($folderId2, $this->cache->get('foo/bar2/asd')['parent']); + + $this->repair->run($this->createMock(IOutput::class)); + + $this->assertTrue($this->cache->inCache('foo2/bar')); + $this->assertTrue($this->cache->inCache('foo2/bar2')); + $this->assertTrue($this->cache->inCache('foo2/bar/asd')); + $this->assertTrue($this->cache->inCache('foo2/bar2/asd')); + $this->assertFalse($this->cache->inCache('foo/bar/asd')); + $this->assertFalse($this->cache->inCache('foo/bar2/asd')); + + $this->assertEquals($folderId1, $this->cache->get('foo2/bar/asd')['parent']); + $this->assertEquals($folderId2, $this->cache->get('foo2/bar2/asd')['parent']); + $this->assertEquals($folderId1, $this->cache->getId('foo2/bar')); + $this->assertEquals($folderId2, $this->cache->getId('foo2/bar2')); + } } From ed6c61256dd1591eeed22a2dcccc640db71a05ee Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 16 Jan 2017 16:16:32 +0100 Subject: [PATCH 07/13] Add MD5() to sqlite Signed-off-by: Robin Appelman --- lib/private/DB/SQLiteSessionInit.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/private/DB/SQLiteSessionInit.php b/lib/private/DB/SQLiteSessionInit.php index f8e6dcef8addd..0e947b9918e6d 100644 --- a/lib/private/DB/SQLiteSessionInit.php +++ b/lib/private/DB/SQLiteSessionInit.php @@ -58,6 +58,9 @@ public function postConnect(ConnectionEventArgs $args) { $sensitive = ($this->caseSensitiveLike) ? 'true' : 'false'; $args->getConnection()->executeUpdate('PRAGMA case_sensitive_like = ' . $sensitive); $args->getConnection()->executeUpdate('PRAGMA journal_mode = ' . $this->journalMode); + /** @var \PDO $pdo */ + $pdo = $args->getConnection()->getWrappedConnection(); + $pdo->sqliteCreateFunction('md5', 'md5', 1); } public function getSubscribedEvents() { From f9eda2706ae4cd297c28a80bb13e1dcd88dc9c8e Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 16 Jan 2017 15:31:04 +0100 Subject: [PATCH 08/13] Add function builder to the query builder Signed-off-by: Robin Appelman --- .../FunctionBuilder/FunctionBuilder.php | 55 ++++++++++++++ .../FunctionBuilder/OCIFunctionBuilder.php | 29 ++++++++ .../FunctionBuilder/PgSqlFunctionBuilder.php | 29 ++++++++ .../FunctionBuilder/SqliteFunctionBuilder.php | 29 ++++++++ lib/private/DB/QueryBuilder/QueryBuilder.php | 33 +++++++++ .../DB/QueryBuilder/IFunctionBuilder.php | 61 +++++++++++++++ lib/public/DB/QueryBuilder/IQueryBuilder.php | 19 +++++ .../DB/QueryBuilder/FunctionBuilderTest.php | 74 +++++++++++++++++++ 8 files changed, 329 insertions(+) create mode 100644 lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php create mode 100644 lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php create mode 100644 lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php create mode 100644 lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php create mode 100644 lib/public/DB/QueryBuilder/IFunctionBuilder.php create mode 100644 tests/lib/DB/QueryBuilder/FunctionBuilderTest.php diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php new file mode 100644 index 0000000000000..61d32d09a62e8 --- /dev/null +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php @@ -0,0 +1,55 @@ + + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\DB\QueryBuilder\FunctionBuilder; + +use OC\DB\QueryBuilder\QueryFunction; +use OC\DB\QueryBuilder\QuoteHelper; +use OCP\DB\QueryBuilder\IFunctionBuilder; + +class FunctionBuilder implements IFunctionBuilder { + /** @var QuoteHelper */ + protected $helper; + + /** + * ExpressionBuilder constructor. + * + * @param QuoteHelper $helper + */ + public function __construct(QuoteHelper $helper) { + $this->helper = $helper; + } + + public function md5($input) { + return new QueryFunction('MD5(' . $this->helper->quoteColumnName($input) . ')'); + } + + public function concat($x, $y) { + return new QueryFunction('CONCAT(' . $this->helper->quoteColumnName($x) . ', ' . $this->helper->quoteColumnName($y) . ')'); + } + + public function substring($input, $start, $length = null) { + if ($length) { + return new QueryFunction('SUBSTR(' . $this->helper->quoteColumnName($input) . ', ' . $this->helper->quoteColumnName($start) . ', ' . $this->helper->quoteColumnName($length) . ')'); + } else { + return new QueryFunction('SUBSTR(' . $this->helper->quoteColumnName($input) . ', ' . $this->helper->quoteColumnName($start) . ')'); + } + } +} diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php new file mode 100644 index 0000000000000..0877d4781ce99 --- /dev/null +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php @@ -0,0 +1,29 @@ + + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\DB\QueryBuilder\FunctionBuilder; + +use OC\DB\QueryBuilder\QueryFunction; + +class OCIFunctionBuilder extends FunctionBuilder { + public function md5($input) { + return new QueryFunction('LOWER(DBMS_OBFUSCATION_TOOLKIT.md5 (input => UTL_RAW.cast_to_raw(' . $this->helper->quoteColumnName($input) .')))'); + } +} diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php new file mode 100644 index 0000000000000..b6a7dce46eb2a --- /dev/null +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php @@ -0,0 +1,29 @@ + + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\DB\QueryBuilder\FunctionBuilder; + +use OC\DB\QueryBuilder\QueryFunction; + +class PgSqlFunctionBuilder extends FunctionBuilder { + public function concat($x, $y) { + return new QueryFunction($this->helper->quoteColumnName($x) . ' || ' . $this->helper->quoteColumnName($y)); + } +} diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php new file mode 100644 index 0000000000000..20b30d0ab3a6e --- /dev/null +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php @@ -0,0 +1,29 @@ + + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\DB\QueryBuilder\FunctionBuilder; + +use OC\DB\QueryBuilder\QueryFunction; + +class SqliteFunctionBuilder extends FunctionBuilder { + public function concat($x, $y) { + return new QueryFunction($this->helper->quoteColumnName($x) . ' || ' . $this->helper->quoteColumnName($y)); + } +} diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index d5dd9cf03661c..5dec529545eb7 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -31,6 +31,11 @@ use OC\DB\QueryBuilder\ExpressionBuilder\MySqlExpressionBuilder; use OC\DB\QueryBuilder\ExpressionBuilder\OCIExpressionBuilder; use OC\DB\QueryBuilder\ExpressionBuilder\PgSqlExpressionBuilder; +use OC\DB\QueryBuilder\ExpressionBuilder\SqliteExpressionBuilder; +use OC\DB\QueryBuilder\FunctionBuilder\FunctionBuilder; +use OC\DB\QueryBuilder\FunctionBuilder\OCIFunctionBuilder; +use OC\DB\QueryBuilder\FunctionBuilder\PgSqlFunctionBuilder; +use OC\DB\QueryBuilder\FunctionBuilder\SqliteFunctionBuilder; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\DB\QueryBuilder\IQueryFunction; @@ -115,6 +120,34 @@ public function expr() { } } + /** + * Gets an FunctionBuilder used for object-oriented construction of query functions. + * This producer method is intended for convenient inline usage. Example: + * + * + * $qb = $conn->getQueryBuilder() + * ->select('u') + * ->from('users', 'u') + * ->where($qb->fun()->md5('u.id')); + * + * + * For more complex function construction, consider storing the function + * builder object in a local variable. + * + * @return \OCP\DB\QueryBuilder\IFunctionBuilder + */ + public function fun() { + if ($this->connection instanceof OracleConnection) { + return new OCIFunctionBuilder($this->helper); + } else if ($this->connection->getDatabasePlatform() instanceof SqlitePlatform) { + return new SqliteFunctionBuilder($this->helper); + } else if ($this->connection->getDatabasePlatform() instanceof PostgreSqlPlatform) { + return new PgSqlFunctionBuilder($this->helper); + } else { + return new FunctionBuilder($this->helper); + } + } + /** * Gets the type of the currently built query. * diff --git a/lib/public/DB/QueryBuilder/IFunctionBuilder.php b/lib/public/DB/QueryBuilder/IFunctionBuilder.php new file mode 100644 index 0000000000000..9798a3a770ee0 --- /dev/null +++ b/lib/public/DB/QueryBuilder/IFunctionBuilder.php @@ -0,0 +1,61 @@ + + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCP\DB\QueryBuilder; + +/** + * This class provides a builder for sql some functions + * + * @since 12.0.0 + */ +interface IFunctionBuilder { + /** + * Calculates the MD5 hash of a given input + * + * @param mixed $input The input to be hashed + * + * @return IQueryFunction + * @since 12.0.0 + */ + public function md5($input); + + /** + * Combines two input strings + * + * @param mixed $x The first input string + * @param mixed $y The seccond input string + * + * @return IQueryFunction + * @since 12.0.0 + */ + public function concat($x, $y); + + /** + * Takes a substring from the input string + * + * @param mixed $input The input string + * @param mixed $start The start of the substring, note that counting starts at 1 + * @param mixed $length The length of the substring + * + * @return IQueryFunction + * @since 12.0.0 + */ + public function substring($input, $start, $length = null); +} diff --git a/lib/public/DB/QueryBuilder/IQueryBuilder.php b/lib/public/DB/QueryBuilder/IQueryBuilder.php index 8ef8a96b25f1a..2a88e7ac5667e 100644 --- a/lib/public/DB/QueryBuilder/IQueryBuilder.php +++ b/lib/public/DB/QueryBuilder/IQueryBuilder.php @@ -94,6 +94,25 @@ public function automaticTablePrefix($enabled); */ public function expr(); + /** + * Gets an FunctionBuilder used for object-oriented construction of query functions. + * This producer method is intended for convenient inline usage. Example: + * + * + * $qb = $conn->getQueryBuilder() + * ->select('u') + * ->from('users', 'u') + * ->where($qb->fun()->md5('u.id')); + * + * + * For more complex function construction, consider storing the function + * builder object in a local variable. + * + * @return \OCP\DB\QueryBuilder\IFunctionBuilder + * @since 12.0.0 + */ + public function fun(); + /** * Gets the type of the currently built query. * diff --git a/tests/lib/DB/QueryBuilder/FunctionBuilderTest.php b/tests/lib/DB/QueryBuilder/FunctionBuilderTest.php new file mode 100644 index 0000000000000..b1ef3d7bb75ec --- /dev/null +++ b/tests/lib/DB/QueryBuilder/FunctionBuilderTest.php @@ -0,0 +1,74 @@ + + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace Test\DB\QueryBuilder; + +use OC\DB\QueryBuilder\Literal; +use Test\TestCase; + +/** + * Class FunctionBuilderTest + * + * @group DB + * + * @package Test\DB\QueryBuilder + */ +class FunctionBuilderTest extends TestCase { + /** @var \Doctrine\DBAL\Connection|\OCP\IDBConnection */ + protected $connection; + + protected function setUp() { + parent::setUp(); + + $this->connection = \OC::$server->getDatabaseConnection(); + } + + public function testConcat() { + $query = $this->connection->getQueryBuilder(); + + $query->select($query->fun()->concat($query->createNamedParameter('foo'), new Literal("'bar'"))); + + $this->assertEquals('foobar', $query->execute()->fetchColumn()); + } + + public function testMd5() { + $query = $this->connection->getQueryBuilder(); + + $query->select($query->fun()->md5($query->createNamedParameter('foobar'))); + + $this->assertEquals(md5('foobar'), $query->execute()->fetchColumn()); + } + + public function testSubstring() { + $query = $this->connection->getQueryBuilder(); + + $query->select($query->fun()->substring($query->createNamedParameter('foobar'), new Literal(2), $query->createNamedParameter(2))); + + $this->assertEquals('oo', $query->execute()->fetchColumn()); + } + + public function testSubstringNoLength() { + $query = $this->connection->getQueryBuilder(); + + $query->select($query->fun()->substring($query->createNamedParameter('foobar'), new Literal(2))); + + $this->assertEquals('oobar', $query->execute()->fetchColumn()); + } +} From d132b80111d350e28f5cc2b21141a40b74e8da63 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Sun, 26 Mar 2017 20:45:49 +0200 Subject: [PATCH 09/13] rename fun to func Signed-off-by: Robin Appelman --- lib/private/DB/QueryBuilder/QueryBuilder.php | 2 +- lib/public/DB/QueryBuilder/IQueryBuilder.php | 2 +- tests/lib/DB/QueryBuilder/FunctionBuilderTest.php | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index 5dec529545eb7..047923fafc214 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -136,7 +136,7 @@ public function expr() { * * @return \OCP\DB\QueryBuilder\IFunctionBuilder */ - public function fun() { + public function func() { if ($this->connection instanceof OracleConnection) { return new OCIFunctionBuilder($this->helper); } else if ($this->connection->getDatabasePlatform() instanceof SqlitePlatform) { diff --git a/lib/public/DB/QueryBuilder/IQueryBuilder.php b/lib/public/DB/QueryBuilder/IQueryBuilder.php index 2a88e7ac5667e..a176bb917a39c 100644 --- a/lib/public/DB/QueryBuilder/IQueryBuilder.php +++ b/lib/public/DB/QueryBuilder/IQueryBuilder.php @@ -111,7 +111,7 @@ public function expr(); * @return \OCP\DB\QueryBuilder\IFunctionBuilder * @since 12.0.0 */ - public function fun(); + public function func(); /** * Gets the type of the currently built query. diff --git a/tests/lib/DB/QueryBuilder/FunctionBuilderTest.php b/tests/lib/DB/QueryBuilder/FunctionBuilderTest.php index b1ef3d7bb75ec..c270e105fc143 100644 --- a/tests/lib/DB/QueryBuilder/FunctionBuilderTest.php +++ b/tests/lib/DB/QueryBuilder/FunctionBuilderTest.php @@ -43,7 +43,7 @@ protected function setUp() { public function testConcat() { $query = $this->connection->getQueryBuilder(); - $query->select($query->fun()->concat($query->createNamedParameter('foo'), new Literal("'bar'"))); + $query->select($query->func()->concat($query->createNamedParameter('foo'), new Literal("'bar'"))); $this->assertEquals('foobar', $query->execute()->fetchColumn()); } @@ -51,7 +51,7 @@ public function testConcat() { public function testMd5() { $query = $this->connection->getQueryBuilder(); - $query->select($query->fun()->md5($query->createNamedParameter('foobar'))); + $query->select($query->func()->md5($query->createNamedParameter('foobar'))); $this->assertEquals(md5('foobar'), $query->execute()->fetchColumn()); } @@ -59,7 +59,7 @@ public function testMd5() { public function testSubstring() { $query = $this->connection->getQueryBuilder(); - $query->select($query->fun()->substring($query->createNamedParameter('foobar'), new Literal(2), $query->createNamedParameter(2))); + $query->select($query->func()->substring($query->createNamedParameter('foobar'), new Literal(2), $query->createNamedParameter(2))); $this->assertEquals('oo', $query->execute()->fetchColumn()); } @@ -67,7 +67,7 @@ public function testSubstring() { public function testSubstringNoLength() { $query = $this->connection->getQueryBuilder(); - $query->select($query->fun()->substring($query->createNamedParameter('foobar'), new Literal(2))); + $query->select($query->func()->substring($query->createNamedParameter('foobar'), new Literal(2))); $this->assertEquals('oobar', $query->execute()->fetchColumn()); } From f1b7dbf46b712d7dbda042b944812ea633cb20aa Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 22 Sep 2017 14:42:08 +0200 Subject: [PATCH 10/13] Fix version for repair step Signed-off-by: Robin Appelman --- lib/composer/composer/autoload_classmap.php | 7 ++++++- lib/composer/composer/autoload_static.php | 7 ++++++- lib/private/Repair/{NC13 => NC11}/RepairInvalidPaths.php | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) rename lib/private/Repair/{NC13 => NC11}/RepairInvalidPaths.php (96%) diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 447b062dc05df..d28af8b453503 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -80,6 +80,7 @@ 'OCP\\DB' => $baseDir . '/lib/public/DB.php', 'OCP\\DB\\QueryBuilder\\ICompositeExpression' => $baseDir . '/lib/public/DB/QueryBuilder/ICompositeExpression.php', 'OCP\\DB\\QueryBuilder\\IExpressionBuilder' => $baseDir . '/lib/public/DB/QueryBuilder/IExpressionBuilder.php', + 'OCP\\DB\\QueryBuilder\\IFunctionBuilder' => $baseDir . '/lib/public/DB/QueryBuilder/IFunctionBuilder.php', 'OCP\\DB\\QueryBuilder\\ILiteral' => $baseDir . '/lib/public/DB/QueryBuilder/ILiteral.php', 'OCP\\DB\\QueryBuilder\\IParameter' => $baseDir . '/lib/public/DB/QueryBuilder/IParameter.php', 'OCP\\DB\\QueryBuilder\\IQueryBuilder' => $baseDir . '/lib/public/DB/QueryBuilder/IQueryBuilder.php', @@ -455,6 +456,10 @@ 'OC\\DB\\QueryBuilder\\ExpressionBuilder\\MySqlExpressionBuilder' => $baseDir . '/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php', 'OC\\DB\\QueryBuilder\\ExpressionBuilder\\OCIExpressionBuilder' => $baseDir . '/lib/private/DB/QueryBuilder/ExpressionBuilder/OCIExpressionBuilder.php', 'OC\\DB\\QueryBuilder\\ExpressionBuilder\\PgSqlExpressionBuilder' => $baseDir . '/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php', + 'OC\\DB\\QueryBuilder\\FunctionBuilder\\FunctionBuilder' => $baseDir . '/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php', + 'OC\\DB\\QueryBuilder\\FunctionBuilder\\OCIFunctionBuilder' => $baseDir . '/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php', + 'OC\\DB\\QueryBuilder\\FunctionBuilder\\PgSqlFunctionBuilder' => $baseDir . '/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php', + 'OC\\DB\\QueryBuilder\\FunctionBuilder\\SqliteFunctionBuilder' => $baseDir . '/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php', 'OC\\DB\\QueryBuilder\\Literal' => $baseDir . '/lib/private/DB/QueryBuilder/Literal.php', 'OC\\DB\\QueryBuilder\\Parameter' => $baseDir . '/lib/private/DB/QueryBuilder/Parameter.php', 'OC\\DB\\QueryBuilder\\QueryBuilder' => $baseDir . '/lib/private/DB/QueryBuilder/QueryBuilder.php', @@ -683,7 +688,7 @@ 'OC\\Repair\\NC11\\FixMountStorages' => $baseDir . '/lib/private/Repair/NC11/FixMountStorages.php', 'OC\\Repair\\NC11\\MoveAvatars' => $baseDir . '/lib/private/Repair/NC11/MoveAvatars.php', 'OC\\Repair\\NC11\\MoveAvatarsBackgroundJob' => $baseDir . '/lib/private/Repair/NC11/MoveAvatarsBackgroundJob.php', - 'OC\\Repair\\NC13\\RepairInvalidPaths' => $baseDir . '/lib/private/Repair/NC13/RepairInvalidPaths.php', + 'OC\\Repair\\NC13\\RepairInvalidPaths' => $baseDir . '/lib/private/Repair/NC11/RepairInvalidPaths.php', 'OC\\Repair\\OldGroupMembershipShares' => $baseDir . '/lib/private/Repair/OldGroupMembershipShares.php', 'OC\\Repair\\Preview' => $baseDir . '/lib/private/Repair/Preview.php', 'OC\\Repair\\RemoveGetETagEntries' => $baseDir . '/lib/private/Repair/RemoveGetETagEntries.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 019f733e30741..86231a49c7b3c 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -110,6 +110,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OCP\\DB' => __DIR__ . '/../../..' . '/lib/public/DB.php', 'OCP\\DB\\QueryBuilder\\ICompositeExpression' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/ICompositeExpression.php', 'OCP\\DB\\QueryBuilder\\IExpressionBuilder' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/IExpressionBuilder.php', + 'OCP\\DB\\QueryBuilder\\IFunctionBuilder' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/IFunctionBuilder.php', 'OCP\\DB\\QueryBuilder\\ILiteral' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/ILiteral.php', 'OCP\\DB\\QueryBuilder\\IParameter' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/IParameter.php', 'OCP\\DB\\QueryBuilder\\IQueryBuilder' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/IQueryBuilder.php', @@ -485,6 +486,10 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\DB\\QueryBuilder\\ExpressionBuilder\\MySqlExpressionBuilder' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/ExpressionBuilder/MySqlExpressionBuilder.php', 'OC\\DB\\QueryBuilder\\ExpressionBuilder\\OCIExpressionBuilder' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/ExpressionBuilder/OCIExpressionBuilder.php', 'OC\\DB\\QueryBuilder\\ExpressionBuilder\\PgSqlExpressionBuilder' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/ExpressionBuilder/PgSqlExpressionBuilder.php', + 'OC\\DB\\QueryBuilder\\FunctionBuilder\\FunctionBuilder' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/FunctionBuilder/FunctionBuilder.php', + 'OC\\DB\\QueryBuilder\\FunctionBuilder\\OCIFunctionBuilder' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php', + 'OC\\DB\\QueryBuilder\\FunctionBuilder\\PgSqlFunctionBuilder' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php', + 'OC\\DB\\QueryBuilder\\FunctionBuilder\\SqliteFunctionBuilder' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php', 'OC\\DB\\QueryBuilder\\Literal' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/Literal.php', 'OC\\DB\\QueryBuilder\\Parameter' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/Parameter.php', 'OC\\DB\\QueryBuilder\\QueryBuilder' => __DIR__ . '/../../..' . '/lib/private/DB/QueryBuilder/QueryBuilder.php', @@ -713,7 +718,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Repair\\NC11\\FixMountStorages' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/FixMountStorages.php', 'OC\\Repair\\NC11\\MoveAvatars' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/MoveAvatars.php', 'OC\\Repair\\NC11\\MoveAvatarsBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/MoveAvatarsBackgroundJob.php', - 'OC\\Repair\\NC13\\RepairInvalidPaths' => __DIR__ . '/../../..' . '/lib/private/Repair/NC13/RepairInvalidPaths.php', + 'OC\\Repair\\NC13\\RepairInvalidPaths' => __DIR__ . '/../../..' . '/lib/private/Repair/NC11/RepairInvalidPaths.php', 'OC\\Repair\\OldGroupMembershipShares' => __DIR__ . '/../../..' . '/lib/private/Repair/OldGroupMembershipShares.php', 'OC\\Repair\\Preview' => __DIR__ . '/../../..' . '/lib/private/Repair/Preview.php', 'OC\\Repair\\RemoveGetETagEntries' => __DIR__ . '/../../..' . '/lib/private/Repair/RemoveGetETagEntries.php', diff --git a/lib/private/Repair/NC13/RepairInvalidPaths.php b/lib/private/Repair/NC11/RepairInvalidPaths.php similarity index 96% rename from lib/private/Repair/NC13/RepairInvalidPaths.php rename to lib/private/Repair/NC11/RepairInvalidPaths.php index cf0b9e7783ef7..959b42e5d6e01 100644 --- a/lib/private/Repair/NC13/RepairInvalidPaths.php +++ b/lib/private/Repair/NC11/RepairInvalidPaths.php @@ -160,8 +160,8 @@ private function repair() { public function run(IOutput $output) { $versionFromBeforeUpdate = $this->config->getSystemValue('version', '0.0.0'); - // was added to 12.0.0.30 and 13.0.0.1 - if (version_compare($versionFromBeforeUpdate, '12.0.0.30', '<') || version_compare($versionFromBeforeUpdate, '13.0.0.0', '==')) { + // was added to 11.0.5.2 + if (version_compare($versionFromBeforeUpdate, '11.0.5.2', '<')) { $count = $this->repair(); $output->info('Repaired ' . $count . ' paths'); From 12df8dd69fbd374faa720164637c4241b06cc201 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 28 Jul 2017 11:34:35 +0200 Subject: [PATCH 11/13] Add brackets around concat statements so comparing the result works as intended Signed-off-by: Joas Schilling --- .../DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php | 2 +- .../DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php index b6a7dce46eb2a..605135ac18789 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/PgSqlFunctionBuilder.php @@ -24,6 +24,6 @@ class PgSqlFunctionBuilder extends FunctionBuilder { public function concat($x, $y) { - return new QueryFunction($this->helper->quoteColumnName($x) . ' || ' . $this->helper->quoteColumnName($y)); + return new QueryFunction('(' . $this->helper->quoteColumnName($x) . ' || ' . $this->helper->quoteColumnName($y) . ')'); } } diff --git a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php index 20b30d0ab3a6e..c6cef9550e59b 100644 --- a/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php +++ b/lib/private/DB/QueryBuilder/FunctionBuilder/SqliteFunctionBuilder.php @@ -24,6 +24,6 @@ class SqliteFunctionBuilder extends FunctionBuilder { public function concat($x, $y) { - return new QueryFunction($this->helper->quoteColumnName($x) . ' || ' . $this->helper->quoteColumnName($y)); + return new QueryFunction('(' . $this->helper->quoteColumnName($x) . ' || ' . $this->helper->quoteColumnName($y) . ')'); } } From 12b02e9785991a6650fead0c53c53a073640b236 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 22 Sep 2017 16:06:57 +0200 Subject: [PATCH 12/13] Fix function builder Signed-off-by: Robin Appelman --- lib/private/DB/QueryBuilder/QueryBuilder.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index 047923fafc214..c835bdb73102a 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -26,6 +26,7 @@ use Doctrine\DBAL\Platforms\MySqlPlatform; use Doctrine\DBAL\Platforms\PostgreSqlPlatform; +use Doctrine\DBAL\Platforms\SqlitePlatform; use OC\DB\OracleConnection; use OC\DB\QueryBuilder\ExpressionBuilder\ExpressionBuilder; use OC\DB\QueryBuilder\ExpressionBuilder\MySqlExpressionBuilder; From 94fe6fdbb18d3e836391b3db93e651d0ce3d963f Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Fri, 22 Sep 2017 16:13:49 +0200 Subject: [PATCH 13/13] adjust test to new version Signed-off-by: Robin Appelman --- tests/lib/Repair/RepairInvalidPathsTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/Repair/RepairInvalidPathsTest.php b/tests/lib/Repair/RepairInvalidPathsTest.php index fe848b620732f..dc4304b7319fb 100644 --- a/tests/lib/Repair/RepairInvalidPathsTest.php +++ b/tests/lib/Repair/RepairInvalidPathsTest.php @@ -48,7 +48,7 @@ protected function setUp() { $config->expects($this->any()) ->method('getSystemValue') ->with('version', '0.0.0') - ->willReturn('12.0.0.0'); + ->willReturn('11.0.0.1'); $this->repair = new RepairInvalidPaths(\OC::$server->getDatabaseConnection(), $config); }