Skip to content

Commit 2efec34

Browse files
icewind1991icewind1991
committed
Add method to storage backends to get directory content with metadata
Currently you need to use `opendir` and then call `getMetadata` for every file, which adds overhead because most storage backends already get the metadata when doing the `opendir`. While storagebackends can (and do) use caching to relief this problem, this adds cache invalidation dificulties and only a limited number of items are generally cached (to prevent memory usage exploding when scanning large storages) With this new methods storage backends can use the child metadata they got from listing the folder to return metadata without having to keep seperate caches. Signed-off-by: Robin Appelman <robin@icewind.nl>
1 parent 4e8bbc4 commit 2efec34

File tree

10 files changed

+147
-47
lines changed

10 files changed

+147
-47
lines changed

‎apps/files_external/lib/Lib/Storage/SMB.php‎

Lines changed: 57 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
use OC\Files\Filesystem;
5656
use OC\Files\Storage\Common;
5757
use OCA\Files_External\Lib\Notify\SMBNotifyHandler;
58+
use OCP\Constants;
5859
use OCP\Files\Notify\IChange;
5960
use OCP\Files\Notify\IRenameChange;
6061
use OCP\Files\Storage\INotifyStorage;
@@ -97,7 +98,7 @@ public function __construct($params) {
9798
if (isset($params['auth'])) {
9899
$auth = $params['auth'];
99100
} else if (isset($params['user']) && isset($params['password']) && isset($params['share'])) {
100-
list($workgroup, $user) = $this->splitUser($params['user']);
101+
[$workgroup, $user] = $this->splitUser($params['user']);
101102
$auth = new BasicAuth($user, $workgroup, $params['password']);
102103
} else {
103104
throw new \Exception('Invalid configuration, no credentials provided');
@@ -206,30 +207,31 @@ protected function throwUnavailable(\Exception $e) {
206207
* @return \Icewind\SMB\IFileInfo[]
207208
* @throws StorageNotAvailableException
208209
*/
209-
protected function getFolderContents($path) {
210+
protected function getFolderContents($path): iterable {
210211
try {
211212
$path = ltrim($this->buildPath($path), '/');
212213
$files = $this->share->dir($path);
213214
foreach ($files as $file) {
214215
$this->statCache[$path . '/' . $file->getName()] = $file;
215216
}
216-
return array_filter($files, function (IFileInfo $file) {
217+
218+
foreach ($files as $file) {
217219
try {
218220
// the isHidden check is done before checking the config boolean to ensure that the metadata is always fetch
219221
// so we trigger the below exceptions where applicable
220222
$hide = $file->isHidden() && !$this->showHidden;
221223
if ($hide) {
222224
$this->logger->debug('hiding hidden file ' . $file->getName());
223225
}
224-
return !$hide;
226+
if (!$hide) {
227+
yield $file;
228+
}
225229
} catch (ForbiddenException $e) {
226230
$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding forbidden entry ' . $file->getName()]);
227-
return false;
228231
} catch (NotFoundException $e) {
229232
$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding not found entry ' . $file->getName()]);
230-
return false;
231233
}
232-
});
234+
}
233235
} catch (ConnectException $e) {
234236
$this->logger->logException($e, ['message' => 'Error while getting folder content']);
235237
throw new StorageNotAvailableException($e->getMessage(), $e->getCode(), $e);
@@ -508,6 +510,46 @@ public function touch($path, $time = null) {
508510
}
509511
}
510512

513+
public function getMetaData($path) {
514+
$fileInfo = $this->getFileInfo($path);
515+
if (!$fileInfo) {
516+
return null;
517+
}
518+
519+
return $this->getMetaDataFromFileInfo($fileInfo);
520+
}
521+
522+
private function getMetaDataFromFileInfo(IFileInfo $fileInfo) {
523+
$permissions = Constants::PERMISSION_READ + Constants::PERMISSION_SHARE;
524+
525+
if (!$fileInfo->isReadOnly()) {
526+
$permissions += Constants::PERMISSION_DELETE;
527+
$permissions += Constants::PERMISSION_UPDATE;
528+
if ($fileInfo->isDirectory()) {
529+
$permissions += Constants::PERMISSION_CREATE;
530+
}
531+
}
532+
533+
$data = [];
534+
if ($fileInfo->isDirectory()) {
535+
$data['mimetype'] = 'httpd/unix-directory';
536+
} else {
537+
$data['mimetype'] = \OC::$server->getMimeTypeDetector()->detectPath($fileInfo->getPath());
538+
}
539+
$data['mtime'] = $fileInfo->getMTime();
540+
if ($fileInfo->isDirectory()) {
541+
$data['size'] = -1; //unknown
542+
} else {
543+
$data['size'] = $fileInfo->getSize();
544+
}
545+
$data['etag'] = $this->getETag($fileInfo->getPath());
546+
$data['storage_mtime'] = $data['mtime'];
547+
$data['permissions'] = $permissions;
548+
$data['name'] = $fileInfo->getName();
549+
550+
return $data;
551+
}
552+
511553
public function opendir($path) {
512554
try {
513555
$files = $this->getFolderContents($path);
@@ -519,10 +561,17 @@ public function opendir($path) {
519561
$names = array_map(function ($info) {
520562
/** @var \Icewind\SMB\IFileInfo $info */
521563
return $info->getName();
522-
}, $files);
564+
}, iterator_to_array($files));
523565
return IteratorDirectory::wrap($names);
524566
}
525567

568+
public function getDirectoryContent($directory): iterable {
569+
$files = $this->getFolderContents($directory);
570+
foreach ($files as $file) {
571+
yield $this->getMetaDataFromFileInfo($file);
572+
}
573+
}
574+
526575
public function filetype($path) {
527576
try {
528577
return $this->getFileInfo($path)->isDirectory() ? 'dir' : 'file';

‎lib/private/Files/Cache/Scanner.php‎

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,11 @@ protected function getData($path) {
126126
* @param int $parentId
127127
* @param array|null|false $cacheData existing data in the cache for the file to be scanned
128128
* @param bool $lock set to false to disable getting an additional read lock during scanning
129+
* @param null $data the metadata for the file, as returned by the storage
129130
* @return array an array of metadata of the scanned file
130-
* @throws \OC\ServerNotAvailableException
131131
* @throws \OCP\Lock\LockedException
132132
*/
133-
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
133+
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
134134
if ($file !== '') {
135135
try {
136136
$this->storage->verifyPath(dirname($file), basename($file));
@@ -149,7 +149,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData =
149149
}
150150

151151
try {
152-
$data = $this->getData($file);
152+
$data = $data ?? $this->getData($file);
153153
} catch (ForbiddenException $e) {
154154
if ($lock) {
155155
if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
@@ -367,26 +367,6 @@ protected function getExistingChildren($folderId) {
367367
return $existingChildren;
368368
}
369369

370-
/**
371-
* Get the children from the storage
372-
*
373-
* @param string $folder
374-
* @return string[]
375-
*/
376-
protected function getNewChildren($folder) {
377-
$children = [];
378-
if ($dh = $this->storage->opendir($folder)) {
379-
if (is_resource($dh)) {
380-
while (($file = readdir($dh)) !== false) {
381-
if (!Filesystem::isIgnoredDir($file)) {
382-
$children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/');
383-
}
384-
}
385-
}
386-
}
387-
return $children;
388-
}
389-
390370
/**
391371
* scan all the files and folders in a folder
392372
*
@@ -426,19 +406,20 @@ protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse
426406
private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
427407
// we put this in it's own function so it cleans up the memory before we start recursing
428408
$existingChildren = $this->getExistingChildren($folderId);
429-
$newChildren = $this->getNewChildren($path);
409+
$newChildren = $this->storage->getDirectoryContent($path);
430410

431411
if ($this->useTransactions) {
432412
\OC::$server->getDatabaseConnection()->beginTransaction();
433413
}
434414

435415
$exceptionOccurred = false;
436416
$childQueue = [];
437-
foreach ($newChildren as $file) {
417+
foreach ($newChildren as $fileMeta) {
418+
$file = $fileMeta['name'];
438419
$child = $path ? $path . '/' . $file : $file;
439420
try {
440421
$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
441-
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock);
422+
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
442423
if ($data) {
443424
if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
444425
$childQueue[$child] = $data['fileid'];

‎lib/private/Files/Storage/Common.php‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ public function copy($path1, $path2) {
235235
} else {
236236
$source = $this->fopen($path1, 'r');
237237
$target = $this->fopen($path2, 'w');
238-
list(, $result) = \OC_Helper::streamCopy($source, $target);
238+
[, $result] = \OC_Helper::streamCopy($source, $target);
239239
if (!$result) {
240240
\OC::$server->getLogger()->warning("Failed to write data while copying $path1 to $path2");
241241
}
@@ -247,7 +247,7 @@ public function copy($path1, $path2) {
247247
public function getMimeType($path) {
248248
if ($this->is_dir($path)) {
249249
return 'httpd/unix-directory';
250-
} elseif ($this->file_exists($path)) {
250+
} else if ($this->file_exists($path)) {
251251
return \OC::$server->getMimeTypeDetector()->detectPath($path);
252252
} else {
253253
return false;
@@ -625,7 +625,7 @@ public function copyFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t
625625
// are not the same as the original one.Once this is fixed we also
626626
// need to adjust the encryption wrapper.
627627
$target = $this->fopen($targetInternalPath, 'w');
628-
list(, $result) = \OC_Helper::streamCopy($source, $target);
628+
[, $result] = \OC_Helper::streamCopy($source, $target);
629629
if ($result and $preserveMtime) {
630630
$this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath));
631631
}
@@ -718,6 +718,7 @@ public function getMetaData($path) {
718718
$data['etag'] = $this->getETag($path);
719719
$data['storage_mtime'] = $data['mtime'];
720720
$data['permissions'] = $permissions;
721+
$data['name'] = basename($path);
721722

722723
return $data;
723724
}
@@ -858,9 +859,18 @@ public function writeStream(string $path, $stream, int $size = null): int {
858859
if (!$target) {
859860
return 0;
860861
}
861-
list($count, $result) = \OC_Helper::streamCopy($stream, $target);
862+
[$count, $result] = \OC_Helper::streamCopy($stream, $target);
862863
fclose($stream);
863864
fclose($target);
864865
return $count;
865866
}
867+
868+
public function getDirectoryContent($directory): iterable {
869+
$dh = $this->opendir($directory);
870+
$basePath = rtrim($directory, '/');
871+
while (($file = readdir($dh)) !== false) {
872+
$childPath = $basePath . '/' . trim($file, '/');
873+
yield $this->getMetaData($childPath);
874+
}
875+
}
866876
}

‎lib/private/Files/Storage/Storage.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,4 +118,22 @@ public function releaseLock($path, $type, ILockingProvider $provider);
118118
* @throws \OCP\Lock\LockedException
119119
*/
120120
public function changeLock($path, $type, ILockingProvider $provider);
121+
122+
/**
123+
* Get the contents of a directory with metadata
124+
*
125+
* @param string $directory
126+
* @return iterable and iterator or array, containing file metadata
127+
*
128+
* The metadata array will contain the following fields
129+
*
130+
* - name
131+
* - mimetype
132+
* - mtime
133+
* - size
134+
* - etag
135+
* - storage_mtime
136+
* - permissions
137+
*/
138+
public function getDirectoryContent($directory): iterable;
121139
}

‎lib/private/Files/Storage/Wrapper/Availability.php‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,4 +461,15 @@ protected function setUnavailable(StorageNotAvailableException $e) {
461461
$this->getStorageCache()->setAvailability(false, $delay);
462462
throw $e;
463463
}
464+
465+
466+
467+
public function getDirectoryContent($directory): iterable {
468+
$this->checkAvailability();
469+
try {
470+
return parent::getDirectoryContent($directory);
471+
} catch (StorageNotAvailableException $e) {
472+
$this->setUnavailable($e);
473+
}
474+
}
464475
}

‎lib/private/Files/Storage/Wrapper/Encoding.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,4 +534,8 @@ public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t
534534
public function getMetaData($path) {
535535
return $this->storage->getMetaData($this->findPathToUse($path));
536536
}
537+
538+
public function getDirectoryContent($directory): iterable {
539+
return $this->storage->getDirectoryContent($this->findPathToUse($directory));
540+
}
537541
}

‎lib/private/Files/Storage/Wrapper/Encryption.php‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -171,15 +171,7 @@ public function filesize($path) {
171171
return $this->storage->filesize($path);
172172
}
173173

174-
/**
175-
* @param string $path
176-
* @return array
177-
*/
178-
public function getMetaData($path) {
179-
$data = $this->storage->getMetaData($path);
180-
if (is_null($data)) {
181-
return null;
182-
}
174+
private function modifyMetaData(array $data): array {
183175
$fullPath = $this->getFullPath($path);
184176
$info = $this->getCache()->get($path);
185177

@@ -200,6 +192,24 @@ public function getMetaData($path) {
200192
return $data;
201193
}
202194

195+
/**
196+
* @param string $path
197+
* @return array
198+
*/
199+
public function getMetaData($path) {
200+
$data = $this->storage->getMetaData($path);
201+
if (is_null($data)) {
202+
return null;
203+
}
204+
return $this->modifyMetaData($data);
205+
}
206+
207+
public function getDirectoryContent($directory): iterable {
208+
foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
209+
yield $this->modifyMetaData($data);;
210+
}
211+
}
212+
203213
/**
204214
* see http://php.net/manual/en/function.file_get_contents.php
205215
*

‎lib/private/Files/Storage/Wrapper/Jail.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -539,4 +539,8 @@ public function writeStream(string $path, $stream, int $size = null): int {
539539
return $count;
540540
}
541541
}
542+
543+
public function getDirectoryContent($directory): iterable {
544+
return $this->getWrapperStorage()->getDirectoryContent($this->getJailedPath($directory));
545+
}
542546
}

‎lib/private/Files/Storage/Wrapper/PermissionsMask.php‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,4 +157,13 @@ public function getScanner($path = '', $storage = null) {
157157
}
158158
return parent::getScanner($path, $storage);
159159
}
160+
161+
public function getDirectoryContent($directory): iterable {
162+
foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
163+
$data['scan_permissions'] = isset($data['scan_permissions']) ? $data['scan_permissions'] : $data['permissions'];
164+
$data['permissions'] &= $this->mask;
165+
166+
yield $data;
167+
}
168+
}
160169
}

‎lib/private/Files/Storage/Wrapper/Wrapper.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -637,4 +637,8 @@ public function writeStream(string $path, $stream, int $size = null): int {
637637
return $count;
638638
}
639639
}
640+
641+
public function getDirectoryContent($directory): iterable {
642+
return $this->getWrapperStorage()->getDirectoryContent($directory);
643+
}
640644
}

0 commit comments

Comments
 (0)