From 38a78ad1153812ce1c02859f8c57aa959710e2af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 23 Oct 2013 12:33:23 +0200 Subject: [PATCH 1/4] backport search in shared files to stable5 --- apps/files_sharing/lib/cache.php | 78 ++++++++++++++++++++++++++++---- lib/public/share.php | 6 +-- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/apps/files_sharing/lib/cache.php b/apps/files_sharing/lib/cache.php index a2b7e22d1b3e..a7201da8aabb 100644 --- a/apps/files_sharing/lib/cache.php +++ b/apps/files_sharing/lib/cache.php @@ -20,6 +20,7 @@ */ namespace OC\Files\Cache; +use OCP\Share_Backend_Collection; /** * Metadata cache for shared files @@ -218,7 +219,36 @@ public function getStatus($file) { * @return array of file data */ public function search($pattern) { - // TODO + + // normalize pattern + $pattern = $this->normalize($pattern); + + $ids = $this->getAll(); + + $files = array(); + + // divide into 1k chunks + $chunks = array_chunk($ids, 1000); + + foreach ($chunks as $chunk) { + $placeholders = join(',', array_fill(0, count($chunk), '?')); + + $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, + `encrypted`, `unencrypted_size`, `etag` + FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `fileid` IN (' . $placeholders . ')'; + + $result = \OC_DB::executeAudited($sql, array_merge(array($pattern), $chunk)); + + while ($row = $result->fetchRow()) { + if (substr($row['path'], 0, 6)==='files/') { + $row['path'] = substr($row['path'],6); // remove 'files/' from path as it's relative to '/Shared' + } + $row['mimetype'] = $this->getMimetype($row['mimetype']); + $row['mimepart'] = $this->getMimetype($row['mimepart']); + $files[] = $row; + } + } + return $files; } /** @@ -236,13 +266,30 @@ public function searchByMime($mimetype) { } $mimetype = $this->getMimetypeId($mimetype); $ids = $this->getAll(); - $placeholders = join(',', array_fill(0, count($ids), '?')); - $query = \OC_DB::prepare(' - SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted` - FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `fileid` IN (' . $placeholders . ')' - ); - $result = $query->execute(array_merge(array($mimetype), $ids)); - return $result->fetchAll(); + + $files = array(); + + // divide into 1k chunks + $chunks = array_chunk($ids, 1000); + + foreach ($chunks as $chunk) { + $placeholders = join(',', array_fill(0, count($ids), '?')); + $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, + `encrypted`, `unencrypted_size`, `etag` + FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `fileid` IN (' . $placeholders . ')'; + + $result = \OC_DB::executeAudited($sql, array_merge(array($mimetype), $chunk)); + + while ($row = $result->fetchRow()) { + if (substr($row['path'], 0, 6)==='files/') { + $row['path'] = substr($row['path'],6); // remove 'files/' from path as it's relative to '/Shared' + } + $row['mimetype'] = $this->getMimetype($row['mimetype']); + $row['mimepart'] = $this->getMimetype($row['mimepart']); + $files[] = $row; + } + } + return $files; } /** @@ -264,7 +311,20 @@ public function calculateFolderSize($path) { * @return int[] */ public function getAll() { - return \OCP\Share::getItemsSharedWith('file', \OC_Share_Backend_File::FORMAT_GET_ALL); + $ids = \OCP\Share::getItemsSharedWith('file', \OC_Share_Backend_File::FORMAT_GET_ALL); + $folderBackend = \OCP\Share::getBackend('folder'); + if ($folderBackend instanceof Share_Backend_Collection) { + foreach ($ids as $file) { + /** @var $folderBackend Share_Backend_Collection */ + $children = $folderBackend->getChildren($file); + foreach ($children as $child) { + $ids[] = (int)$child['source']; + } + + } + } + + return $ids; } } diff --git a/lib/public/share.php b/lib/public/share.php index d75699334584..7ba5fa502469 100644 --- a/lib/public/share.php +++ b/lib/public/share.php @@ -760,10 +760,10 @@ public static function setExpirationDate($itemType, $itemSource, $date) { /** * @brief Get the backend class for the specified item type - * @param string Item type - * @return Sharing backend object + * @param string $itemType + * @return Share_Backend */ - private static function getBackend($itemType) { + public static function getBackend($itemType) { if (isset(self::$backends[$itemType])) { return self::$backends[$itemType]; } else if (isset(self::$backendTypes[$itemType]['class'])) { From 1e5257c610b202b56d1cf51bb77646497e7cced0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 23 Oct 2013 14:40:47 +0200 Subject: [PATCH 2/4] OC5 does not have executeAudited --- apps/files_sharing/lib/cache.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/files_sharing/lib/cache.php b/apps/files_sharing/lib/cache.php index a7201da8aabb..6e2fae77dffa 100644 --- a/apps/files_sharing/lib/cache.php +++ b/apps/files_sharing/lib/cache.php @@ -237,7 +237,9 @@ public function search($pattern) { `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `fileid` IN (' . $placeholders . ')'; - $result = \OC_DB::executeAudited($sql, array_merge(array($pattern), $chunk)); + $stmt = \OC_DB::prepare($sql); + + $result = $stmt->execute(array_merge(array($pattern), $chunk)); while ($row = $result->fetchRow()) { if (substr($row['path'], 0, 6)==='files/') { @@ -278,7 +280,9 @@ public function searchByMime($mimetype) { `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `fileid` IN (' . $placeholders . ')'; - $result = \OC_DB::executeAudited($sql, array_merge(array($mimetype), $chunk)); + $stmt = \OC_DB::prepare($sql); + + $result = $stmt->execute(array_merge(array($mimetype), $chunk)); while ($row = $result->fetchRow()) { if (substr($row['path'], 0, 6)==='files/') { From 149679ca68b21b291f55408067fad3bcead4ca4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 29 Oct 2013 13:34:53 +0100 Subject: [PATCH 3/4] cleanup PR - extract method, - introduce MAX CHUNK SIZE const with 999 (reduce this by the number of additional query parameters), - fix whitespace --- apps/files_sharing/lib/cache.php | 79 +++++++++++++++++--------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/apps/files_sharing/lib/cache.php b/apps/files_sharing/lib/cache.php index 6e2fae77dffa..f40ab1efdc0c 100644 --- a/apps/files_sharing/lib/cache.php +++ b/apps/files_sharing/lib/cache.php @@ -220,73 +220,78 @@ public function getStatus($file) { */ public function search($pattern) { - // normalize pattern - $pattern = $this->normalize($pattern); - - $ids = $this->getAll(); - - $files = array(); - - // divide into 1k chunks - $chunks = array_chunk($ids, 1000); + $where = '`name` LIKE ? AND '; - foreach ($chunks as $chunk) { - $placeholders = join(',', array_fill(0, count($chunk), '?')); + // normalize pattern + $value = $this->normalize($pattern); - $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, - `encrypted`, `unencrypted_size`, `etag` - FROM `*PREFIX*filecache` WHERE `name` LIKE ? AND `fileid` IN (' . $placeholders . ')'; + // we have one ? in our where clause + $chunksize = self::MAX_SQL_CHUNK_SIZE - 1; - $stmt = \OC_DB::prepare($sql); - - $result = $stmt->execute(array_merge(array($pattern), $chunk)); + return $this->searchWithWhere($where, $value, $chunksize); - while ($row = $result->fetchRow()) { - if (substr($row['path'], 0, 6)==='files/') { - $row['path'] = substr($row['path'],6); // remove 'files/' from path as it's relative to '/Shared' - } - $row['mimetype'] = $this->getMimetype($row['mimetype']); - $row['mimepart'] = $this->getMimetype($row['mimepart']); - $files[] = $row; - } - } - return $files; } /** * search for files by mimetype * - * @param string $part1 - * @param string $part2 + * @param string $mimetype * @return array */ public function searchByMime($mimetype) { + if (strpos($mimetype, '/')) { - $where = '`mimetype` = ?'; + $where = '`mimetype` = ? AND '; } else { - $where = '`mimepart` = ?'; + $where = '`mimepart` = ? AND '; } - $mimetype = $this->getMimetypeId($mimetype); + + $value = $this->getMimetypeId($mimetype); + + // we have one ? in our where clause + $chunksize = self::MAX_SQL_CHUNK_SIZE - 1; + + return $this->searchWithWhere($where, $value, $chunksize); + + } + + /** + * The maximum number of placeholders that can be used in an SQL query. + * Value MUST be < 1000. + * Also see ORA-01795 maximum number of expressions in a list is 1000 + */ + const MAX_SQL_CHUNK_SIZE = 999; + + /** + * search for files with a custom where clause and value + * the $wherevalue will be array_merge()d with the file id chunks + * + * @param string $sqlwhere + * @param string $wherevalue + * @return array + */ + private function searchWithWhere($sqlwhere, $wherevalue, $chunksize = self::MAX_SQL_CHUNK_SIZE) { + $ids = $this->getAll(); $files = array(); // divide into 1k chunks - $chunks = array_chunk($ids, 1000); + $chunks = array_chunk($ids, $chunksize); foreach ($chunks as $chunk) { $placeholders = join(',', array_fill(0, count($ids), '?')); $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `unencrypted_size`, `etag` - FROM `*PREFIX*filecache` WHERE ' . $where . ' AND `fileid` IN (' . $placeholders . ')'; + FROM `*PREFIX*filecache` WHERE ' . $sqlwhere . ' `fileid` IN (' . $placeholders . ')'; $stmt = \OC_DB::prepare($sql); - $result = $stmt->execute(array_merge(array($mimetype), $chunk)); + $result = $stmt->execute(array_merge(array($wherevalue), $chunk)); while ($row = $result->fetchRow()) { - if (substr($row['path'], 0, 6)==='files/') { - $row['path'] = substr($row['path'],6); // remove 'files/' from path as it's relative to '/Shared' + if (substr($row['path'], 0, 6) === 'files/') { + $row['path'] = substr($row['path'], 6); // remove 'files/' from path as it's relative to '/Shared' } $row['mimetype'] = $this->getMimetype($row['mimetype']); $row['mimepart'] = $this->getMimetype($row['mimepart']); From a75017bb484674e8edbb4fafe0a2a70b44c654f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 30 Oct 2013 15:02:35 +0100 Subject: [PATCH 4/4] fix placholder counting --- apps/files_sharing/lib/cache.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/files_sharing/lib/cache.php b/apps/files_sharing/lib/cache.php index f40ab1efdc0c..baa632ec9c18 100644 --- a/apps/files_sharing/lib/cache.php +++ b/apps/files_sharing/lib/cache.php @@ -280,7 +280,7 @@ private function searchWithWhere($sqlwhere, $wherevalue, $chunksize = self::MAX_ $chunks = array_chunk($ids, $chunksize); foreach ($chunks as $chunk) { - $placeholders = join(',', array_fill(0, count($ids), '?')); + $placeholders = join(',', array_fill(0, count($chunk), '?')); $sql = 'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted`, `unencrypted_size`, `etag` FROM `*PREFIX*filecache` WHERE ' . $sqlwhere . ' `fileid` IN (' . $placeholders . ')';