Skip to content

Commit 1055e22

Browse files
icewind1991icewind1991
committed
Add method to storage backends to get directory content with metadata
Currently you need to use `opendir` and then call `getMetadata` for every file, which adds overhead because most storage backends already get the metadata when doing the `opendir`. While storagebackends can (and do) use caching to relief this problem, this adds cache invalidation dificulties and only a limited number of items are generally cached (to prevent memory usage exploding when scanning large storages) With this new methods storage backends can use the child metadata they got from listing the folder to return metadata without having to keep seperate caches. Signed-off-by: Robin Appelman <robin@icewind.nl>
1 parent dc9df2b commit 1055e22

File tree

10 files changed

+147
-47
lines changed

10 files changed

+147
-47
lines changed

‎apps/files_external/lib/Lib/Storage/SMB.php‎

Lines changed: 57 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
use OC\Files\Filesystem;
5555
use OC\Files\Storage\Common;
5656
use OCA\Files_External\Lib\Notify\SMBNotifyHandler;
57+
use OCP\Constants;
5758
use OCP\Files\Notify\IChange;
5859
use OCP\Files\Notify\IRenameChange;
5960
use OCP\Files\Storage\INotifyStorage;
@@ -96,7 +97,7 @@ public function __construct($params) {
9697
if (isset($params['auth'])) {
9798
$auth = $params['auth'];
9899
} else if (isset($params['user']) && isset($params['password']) && isset($params['share'])) {
99-
list($workgroup, $user) = $this->splitUser($params['user']);
100+
[$workgroup, $user] = $this->splitUser($params['user']);
100101
$auth = new BasicAuth($user, $workgroup, $params['password']);
101102
} else {
102103
throw new \Exception('Invalid configuration, no credentials provided');
@@ -205,30 +206,31 @@ protected function throwUnavailable(\Exception $e) {
205206
* @return \Icewind\SMB\IFileInfo[]
206207
* @throws StorageNotAvailableException
207208
*/
208-
protected function getFolderContents($path) {
209+
protected function getFolderContents($path): iterable {
209210
try {
210211
$path = ltrim($this->buildPath($path), '/');
211212
$files = $this->share->dir($path);
212213
foreach ($files as $file) {
213214
$this->statCache[$path . '/' . $file->getName()] = $file;
214215
}
215-
return array_filter($files, function (IFileInfo $file) {
216+
217+
foreach ($files as $file) {
216218
try {
217219
// the isHidden check is done before checking the config boolean to ensure that the metadata is always fetch
218220
// so we trigger the below exceptions where applicable
219221
$hide = $file->isHidden() && !$this->showHidden;
220222
if ($hide) {
221223
$this->logger->debug('hiding hidden file ' . $file->getName());
222224
}
223-
return !$hide;
225+
if (!$hide) {
226+
yield $file;
227+
}
224228
} catch (ForbiddenException $e) {
225229
$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding forbidden entry ' . $file->getName()]);
226-
return false;
227230
} catch (NotFoundException $e) {
228231
$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding not found entry ' . $file->getName()]);
229-
return false;
230232
}
231-
});
233+
}
232234
} catch (ConnectException $e) {
233235
$this->logger->logException($e, ['message' => 'Error while getting folder content']);
234236
throw new StorageNotAvailableException($e->getMessage(), $e->getCode(), $e);
@@ -507,6 +509,46 @@ public function touch($path, $time = null) {
507509
}
508510
}
509511

512+
public function getMetaData($path) {
513+
$fileInfo = $this->getFileInfo($path);
514+
if (!$fileInfo) {
515+
return null;
516+
}
517+
518+
return $this->getMetaDataFromFileInfo($fileInfo);
519+
}
520+
521+
private function getMetaDataFromFileInfo(IFileInfo $fileInfo) {
522+
$permissions = Constants::PERMISSION_READ + Constants::PERMISSION_SHARE;
523+
524+
if (!$fileInfo->isReadOnly()) {
525+
$permissions += Constants::PERMISSION_DELETE;
526+
$permissions += Constants::PERMISSION_UPDATE;
527+
if ($fileInfo->isDirectory()) {
528+
$permissions += Constants::PERMISSION_CREATE;
529+
}
530+
}
531+
532+
$data = [];
533+
if ($fileInfo->isDirectory()) {
534+
$data['mimetype'] = 'httpd/unix-directory';
535+
} else {
536+
$data['mimetype'] = \OC::$server->getMimeTypeDetector()->detectPath($fileInfo->getPath());
537+
}
538+
$data['mtime'] = $fileInfo->getMTime();
539+
if ($fileInfo->isDirectory()) {
540+
$data['size'] = -1; //unknown
541+
} else {
542+
$data['size'] = $fileInfo->getSize();
543+
}
544+
$data['etag'] = $this->getETag($fileInfo->getPath());
545+
$data['storage_mtime'] = $data['mtime'];
546+
$data['permissions'] = $permissions;
547+
$data['name'] = $fileInfo->getName();
548+
549+
return $data;
550+
}
551+
510552
public function opendir($path) {
511553
try {
512554
$files = $this->getFolderContents($path);
@@ -518,10 +560,17 @@ public function opendir($path) {
518560
$names = array_map(function ($info) {
519561
/** @var \Icewind\SMB\IFileInfo $info */
520562
return $info->getName();
521-
}, $files);
563+
}, iterator_to_array($files));
522564
return IteratorDirectory::wrap($names);
523565
}
524566

567+
public function getDirectoryContent($directory): iterable {
568+
$files = $this->getFolderContents($directory);
569+
foreach ($files as $file) {
570+
yield $this->getMetaDataFromFileInfo($file);
571+
}
572+
}
573+
525574
public function filetype($path) {
526575
try {
527576
return $this->getFileInfo($path)->isDirectory() ? 'dir' : 'file';

‎lib/private/Files/Cache/Scanner.php‎

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -125,11 +125,11 @@ protected function getData($path) {
125125
* @param int $parentId
126126
* @param array | null $cacheData existing data in the cache for the file to be scanned
127127
* @param bool $lock set to false to disable getting an additional read lock during scanning
128+
* @param null $data the metadata for the file, as returned by the storage
128129
* @return array an array of metadata of the scanned file
129-
* @throws \OC\ServerNotAvailableException
130130
* @throws \OCP\Lock\LockedException
131131
*/
132-
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
132+
public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
133133
if ($file !== '') {
134134
try {
135135
$this->storage->verifyPath(dirname($file), basename($file));
@@ -148,7 +148,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData =
148148
}
149149

150150
try {
151-
$data = $this->getData($file);
151+
$data = $data ?? $this->getData($file);
152152
} catch (ForbiddenException $e) {
153153
if ($lock) {
154154
if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
@@ -365,26 +365,6 @@ protected function getExistingChildren($folderId) {
365365
return $existingChildren;
366366
}
367367

368-
/**
369-
* Get the children from the storage
370-
*
371-
* @param string $folder
372-
* @return string[]
373-
*/
374-
protected function getNewChildren($folder) {
375-
$children = [];
376-
if ($dh = $this->storage->opendir($folder)) {
377-
if (is_resource($dh)) {
378-
while (($file = readdir($dh)) !== false) {
379-
if (!Filesystem::isIgnoredDir($file)) {
380-
$children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/');
381-
}
382-
}
383-
}
384-
}
385-
return $children;
386-
}
387-
388368
/**
389369
* scan all the files and folders in a folder
390370
*
@@ -424,19 +404,20 @@ protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse
424404
private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
425405
// we put this in it's own function so it cleans up the memory before we start recursing
426406
$existingChildren = $this->getExistingChildren($folderId);
427-
$newChildren = $this->getNewChildren($path);
407+
$newChildren = $this->storage->getDirectoryContent($path);
428408

429409
if ($this->useTransactions) {
430410
\OC::$server->getDatabaseConnection()->beginTransaction();
431411
}
432412

433413
$exceptionOccurred = false;
434414
$childQueue = [];
435-
foreach ($newChildren as $file) {
415+
foreach ($newChildren as $fileMeta) {
416+
$file = $fileMeta['name'];
436417
$child = $path ? $path . '/' . $file : $file;
437418
try {
438419
$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : null;
439-
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock);
420+
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
440421
if ($data) {
441422
if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
442423
$childQueue[$child] = $data['fileid'];

‎lib/private/Files/Storage/Common.php‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ public function copy($path1, $path2) {
234234
} else {
235235
$source = $this->fopen($path1, 'r');
236236
$target = $this->fopen($path2, 'w');
237-
list(, $result) = \OC_Helper::streamCopy($source, $target);
237+
[, $result] = \OC_Helper::streamCopy($source, $target);
238238
if (!$result) {
239239
\OC::$server->getLogger()->warning("Failed to write data while copying $path1 to $path2");
240240
}
@@ -246,7 +246,7 @@ public function copy($path1, $path2) {
246246
public function getMimeType($path) {
247247
if ($this->is_dir($path)) {
248248
return 'httpd/unix-directory';
249-
} elseif ($this->file_exists($path)) {
249+
} else if ($this->file_exists($path)) {
250250
return \OC::$server->getMimeTypeDetector()->detectPath($path);
251251
} else {
252252
return false;
@@ -624,7 +624,7 @@ public function copyFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t
624624
// are not the same as the original one.Once this is fixed we also
625625
// need to adjust the encryption wrapper.
626626
$target = $this->fopen($targetInternalPath, 'w');
627-
list(, $result) = \OC_Helper::streamCopy($source, $target);
627+
[, $result] = \OC_Helper::streamCopy($source, $target);
628628
if ($result and $preserveMtime) {
629629
$this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath));
630630
}
@@ -717,6 +717,7 @@ public function getMetaData($path) {
717717
$data['etag'] = $this->getETag($path);
718718
$data['storage_mtime'] = $data['mtime'];
719719
$data['permissions'] = $permissions;
720+
$data['name'] = basename($path);
720721

721722
return $data;
722723
}
@@ -857,9 +858,18 @@ public function writeStream(string $path, $stream, int $size = null): int {
857858
if (!$target) {
858859
return 0;
859860
}
860-
list($count, $result) = \OC_Helper::streamCopy($stream, $target);
861+
[$count, $result] = \OC_Helper::streamCopy($stream, $target);
861862
fclose($stream);
862863
fclose($target);
863864
return $count;
864865
}
866+
867+
public function getDirectoryContent($directory): iterable {
868+
$dh = $this->opendir($directory);
869+
$basePath = rtrim($directory, '/');
870+
while (($file = readdir($dh)) !== false) {
871+
$childPath = $basePath . '/' . trim($file, '/');
872+
yield $this->getMetaData($childPath);
873+
}
874+
}
865875
}

‎lib/private/Files/Storage/Storage.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,4 +118,22 @@ public function releaseLock($path, $type, ILockingProvider $provider);
118118
* @throws \OCP\Lock\LockedException
119119
*/
120120
public function changeLock($path, $type, ILockingProvider $provider);
121+
122+
/**
123+
* Get the contents of a directory with metadata
124+
*
125+
* @param string $directory
126+
* @return iterable and iterator or array, containing file metadata
127+
*
128+
* The metadata array will contain the following fields
129+
*
130+
* - name
131+
* - mimetype
132+
* - mtime
133+
* - size
134+
* - etag
135+
* - storage_mtime
136+
* - permissions
137+
*/
138+
public function getDirectoryContent($directory): iterable;
121139
}

‎lib/private/Files/Storage/Wrapper/Availability.php‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,4 +461,15 @@ protected function setUnavailable(StorageNotAvailableException $e) {
461461
$this->getStorageCache()->setAvailability(false, $delay);
462462
throw $e;
463463
}
464+
465+
466+
467+
public function getDirectoryContent($directory): iterable {
468+
$this->checkAvailability();
469+
try {
470+
return parent::getDirectoryContent($directory);
471+
} catch (StorageNotAvailableException $e) {
472+
$this->setUnavailable($e);
473+
}
474+
}
464475
}

‎lib/private/Files/Storage/Wrapper/Encoding.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,4 +534,8 @@ public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t
534534
public function getMetaData($path) {
535535
return $this->storage->getMetaData($this->findPathToUse($path));
536536
}
537+
538+
public function getDirectoryContent($directory): iterable {
539+
return $this->storage->getDirectoryContent($this->findPathToUse($directory));
540+
}
537541
}

‎lib/private/Files/Storage/Wrapper/Encryption.php‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -170,15 +170,7 @@ public function filesize($path) {
170170
return $this->storage->filesize($path);
171171
}
172172

173-
/**
174-
* @param string $path
175-
* @return array
176-
*/
177-
public function getMetaData($path) {
178-
$data = $this->storage->getMetaData($path);
179-
if (is_null($data)) {
180-
return null;
181-
}
173+
private function modifyMetaData(array $data): array {
182174
$fullPath = $this->getFullPath($path);
183175
$info = $this->getCache()->get($path);
184176

@@ -199,6 +191,24 @@ public function getMetaData($path) {
199191
return $data;
200192
}
201193

194+
/**
195+
* @param string $path
196+
* @return array
197+
*/
198+
public function getMetaData($path) {
199+
$data = $this->storage->getMetaData($path);
200+
if (is_null($data)) {
201+
return null;
202+
}
203+
return $this->modifyMetaData($data);
204+
}
205+
206+
public function getDirectoryContent($directory): iterable {
207+
foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
208+
yield $this->modifyMetaData($data);;
209+
}
210+
}
211+
202212
/**
203213
* see http://php.net/manual/en/function.file_get_contents.php
204214
*

‎lib/private/Files/Storage/Wrapper/Jail.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -539,4 +539,8 @@ public function writeStream(string $path, $stream, int $size = null): int {
539539
return $count;
540540
}
541541
}
542+
543+
public function getDirectoryContent($directory): iterable {
544+
return $this->getWrapperStorage()->getDirectoryContent($this->getJailedPath($directory));
545+
}
542546
}

‎lib/private/Files/Storage/Wrapper/PermissionsMask.php‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,4 +157,13 @@ public function getScanner($path = '', $storage = null) {
157157
}
158158
return parent::getScanner($path, $storage);
159159
}
160+
161+
public function getDirectoryContent($directory): iterable {
162+
foreach ($this->getWrapperStorage()->getDirectoryContent($directory) as $data) {
163+
$data['scan_permissions'] = isset($data['scan_permissions']) ? $data['scan_permissions'] : $data['permissions'];
164+
$data['permissions'] &= $this->mask;
165+
166+
yield $data;
167+
}
168+
}
160169
}

‎lib/private/Files/Storage/Wrapper/Wrapper.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,4 +636,8 @@ public function writeStream(string $path, $stream, int $size = null): int {
636636
return $count;
637637
}
638638
}
639+
640+
public function getDirectoryContent($directory): iterable {
641+
return $this->getWrapperStorage()->getDirectoryContent($directory);
642+
}
639643
}

0 commit comments

Comments
 (0)