Add method to storage backends to get directory content with metadata

Currently you need to use `opendir` and then call `getMetadata` for
every file, which adds overhead because most storage backends already
get the metadata when doing the `opendir`.

While storagebackends can (and do) use caching to relief this problem,
this adds cache invalidation dificulties and only a limited number of
items are generally cached (to prevent memory usage exploding when
scanning large storages)

With this new methods storage backends can use the child metadata they
got from listing the folder to return metadata without having to keep
seperate caches.

Signed-off-by: Robin Appelman <robin@icewind.nl>

Author: icewind1991

apps/files_external/lib/Lib/Storage/SMB.php

@@ -55,6 +55,7 @@
 use OC\Files\Filesystem;
 use OC\Files\Storage\Common;
 use OCA\Files_External\Lib\Notify\SMBNotifyHandler;
+use OCP\Constants;
 use OCP\Files\Notify\IChange;
 use OCP\Files\Notify\IRenameChange;
 use OCP\Files\Storage\INotifyStorage;
@@ -97,7 +98,7 @@ public function __construct($params) {
 		if (isset($params['auth'])) {
 			$auth = $params['auth'];
 		} elseif (isset($params['user']) && isset($params['password']) && isset($params['share'])) {
-			list($workgroup, $user) = $this->splitUser($params['user']);
+			[$workgroup, $user] = $this->splitUser($params['user']);
 			$auth = new BasicAuth($user, $workgroup, $params['password']);
 		} else {
 			throw new \Exception('Invalid configuration, no credentials provided');
@@ -206,30 +207,31 @@ protected function throwUnavailable(\Exception $e) {
 	 * @return \Icewind\SMB\IFileInfo[]
 	 * @throws StorageNotAvailableException
 	 */
-	protected function getFolderContents($path) {
+	protected function getFolderContents($path): iterable {
 		try {
 			$path = ltrim($this->buildPath($path), '/');
 			$files = $this->share->dir($path);
 			foreach ($files as $file) {
 				$this->statCache[$path . '/' . $file->getName()] = $file;
 			}
-			return array_filter($files, function (IFileInfo $file) {
+
+			foreach ($files as $file) {
 				try {
 					// the isHidden check is done before checking the config boolean to ensure that the metadata is always fetch
 					// so we trigger the below exceptions where applicable
 					$hide = $file->isHidden() && !$this->showHidden;
 					if ($hide) {
 						$this->logger->debug('hiding hidden file ' . $file->getName());
 					}
-					return !$hide;
+					if (!$hide) {
+						yield $file;
+					}
 				} catch (ForbiddenException $e) {
 					$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding forbidden entry ' . $file->getName()]);
-					return false;
 				} catch (NotFoundException $e) {
 					$this->logger->logException($e, ['level' => ILogger::DEBUG, 'message' => 'Hiding not found entry ' . $file->getName()]);
-					return false;
 				}
-			});
+			}
 		} catch (ConnectException $e) {
 			$this->logger->logException($e, ['message' => 'Error while getting folder content']);
 			throw new StorageNotAvailableException($e->getMessage(), $e->getCode(), $e);
@@ -508,6 +510,46 @@ public function touch($path, $time = null) {
 		}
 	}
 
+	public function getMetaData($path) {
+		$fileInfo = $this->getFileInfo($path);
+		if (!$fileInfo) {
+			return null;
+		}
+
+		return $this->getMetaDataFromFileInfo($fileInfo);
+	}
+
+	private function getMetaDataFromFileInfo(IFileInfo $fileInfo) {
+		$permissions = Constants::PERMISSION_READ + Constants::PERMISSION_SHARE;
+
+		if (!$fileInfo->isReadOnly()) {
+			$permissions += Constants::PERMISSION_DELETE;
+			$permissions += Constants::PERMISSION_UPDATE;
+			if ($fileInfo->isDirectory()) {
+				$permissions += Constants::PERMISSION_CREATE;
+			}
+		}
+
+		$data = [];
+		if ($fileInfo->isDirectory()) {
+			$data['mimetype'] = 'httpd/unix-directory';
+		} else {
+			$data['mimetype'] = \OC::$server->getMimeTypeDetector()->detectPath($fileInfo->getPath());
+		}
+		$data['mtime'] = $fileInfo->getMTime();
+		if ($fileInfo->isDirectory()) {
+			$data['size'] = -1; //unknown
+		} else {
+			$data['size'] = $fileInfo->getSize();
+		}
+		$data['etag'] = $this->getETag($fileInfo->getPath());
+		$data['storage_mtime'] = $data['mtime'];
+		$data['permissions'] = $permissions;
+		$data['name'] = $fileInfo->getName();
+
+		return $data;
+	}
+
 	public function opendir($path) {
 		try {
 			$files = $this->getFolderContents($path);
@@ -519,10 +561,17 @@ public function opendir($path) {
 		$names = array_map(function ($info) {
 			/** @var \Icewind\SMB\IFileInfo $info */
 			return $info->getName();
-		}, $files);
+		}, iterator_to_array($files));
 		return IteratorDirectory::wrap($names);
 	}
 
+	public function getDirectoryContent($directory): \Traversable {
+		$files = $this->getFolderContents($directory);
+		foreach ($files as $file) {
+			yield $this->getMetaDataFromFileInfo($file);
+		}
+	}
+
 	public function filetype($path) {
 		try {
 			return $this->getFileInfo($path)->isDirectory() ? 'dir' : 'file';

apps/files_sharing/lib/External/Scanner.php

@@ -57,7 +57,7 @@ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $loc
 	 * @param bool $lock set to false to disable getting an additional read lock during scanning
 	 * @return array an array of metadata of the scanned file
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		try {
 			return parent::scanFile($file, $reuseExisting);
 		} catch (ForbiddenException $e) {

apps/files_sharing/lib/Scanner.php

@@ -71,7 +71,7 @@ private function getSourceScanner() {
 		}
 	}
 
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		$sourceScanner = $this->getSourceScanner();
 		if ($sourceScanner instanceof NoopScanner) {
 			return [];

lib/private/Files/Cache/Scanner.php

@@ -126,11 +126,11 @@ protected function getData($path) {
 	 * @param int $parentId
 	 * @param array|null|false $cacheData existing data in the cache for the file to be scanned
 	 * @param bool $lock set to false to disable getting an additional read lock during scanning
+	 * @param null $data the metadata for the file, as returned by the storage
 	 * @return array an array of metadata of the scanned file
-	 * @throws \OC\ServerNotAvailableException
 	 * @throws \OCP\Lock\LockedException
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		if ($file !== '') {
 			try {
 				$this->storage->verifyPath(dirname($file), basename($file));
@@ -149,7 +149,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData =
 			}
 
 			try {
-				$data = $this->getData($file);
+				$data = $data ?? $this->getData($file);
 			} catch (ForbiddenException $e) {
 				if ($lock) {
 					if ($this->storage->instanceOfStorage('\OCP\Files\Storage\ILockingStorage')) {
@@ -366,26 +366,6 @@ protected function getExistingChildren($folderId) {
 		return $existingChildren;
 	}
 
-	/**
-	 * Get the children from the storage
-	 *
-	 * @param string $folder
-	 * @return string[]
-	 */
-	protected function getNewChildren($folder) {
-		$children = [];
-		if ($dh = $this->storage->opendir($folder)) {
-			if (is_resource($dh)) {
-				while (($file = readdir($dh)) !== false) {
-					if (!Filesystem::isIgnoredDir($file)) {
-						$children[] = trim(\OC\Files\Filesystem::normalizePath($file), '/');
-					}
-				}
-			}
-		}
-		return $children;
-	}
-
 	/**
 	 * scan all the files and folders in a folder
 	 *
@@ -425,19 +405,22 @@ protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse
 	private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$size) {
 		// we put this in it's own function so it cleans up the memory before we start recursing
 		$existingChildren = $this->getExistingChildren($folderId);
-		$newChildren = $this->getNewChildren($path);
+		$newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
 
 		if ($this->useTransactions) {
 			\OC::$server->getDatabaseConnection()->beginTransaction();
 		}
 
 		$exceptionOccurred = false;
 		$childQueue = [];
-		foreach ($newChildren as $file) {
+		$newChildNames = [];
+		foreach ($newChildren as $fileMeta) {
+			$file = $fileMeta['name'];
+			$newChildNames[] = $file;
 			$child = $path ? $path . '/' . $file : $file;
 			try {
 				$existingData = isset($existingChildren[$file]) ? $existingChildren[$file] : false;
-				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock);
+				$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
 				if ($data) {
 					if ($data['mimetype'] === 'httpd/unix-directory' and $recursive === self::SCAN_RECURSIVE) {
 						$childQueue[$child] = $data['fileid'];
@@ -471,7 +454,7 @@ private function handleChildren($path, $recursive, $reuse, $folderId, $lock, &$s
 				throw $e;
 			}
 		}
-		$removedChildren = \array_diff(array_keys($existingChildren), $newChildren);
+		$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
 		foreach ($removedChildren as $childName) {
 			$child = $path ? $path . '/' . $childName : $childName;
 			$this->removeFromCache($child);

lib/private/Files/ObjectStore/NoopScanner.php

@@ -44,7 +44,7 @@ public function __construct(Storage $storage) {
 	 * @param array|null $cacheData existing data in the cache for the file to be scanned
 	 * @return array an array of metadata of the scanned file
 	 */
-	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true) {
+	public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) {
 		return [];
 	}
 

lib/private/Files/Storage/Common.php

@@ -234,7 +234,7 @@ public function copy($path1, $path2) {
 		} else {
 			$source = $this->fopen($path1, 'r');
 			$target = $this->fopen($path2, 'w');
-			list(, $result) = \OC_Helper::streamCopy($source, $target);
+			[, $result] = \OC_Helper::streamCopy($source, $target);
 			if (!$result) {
 				\OC::$server->getLogger()->warning("Failed to write data while copying $path1 to $path2");
 			}
@@ -246,7 +246,7 @@ public function copy($path1, $path2) {
 	public function getMimeType($path) {
 		if ($this->is_dir($path)) {
 			return 'httpd/unix-directory';
-		} elseif ($this->file_exists($path)) {
+		} else if ($this->file_exists($path)) {
 			return \OC::$server->getMimeTypeDetector()->detectPath($path);
 		} else {
 			return false;
@@ -626,7 +626,7 @@ public function copyFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t
 			// are not the same as the original one.Once this is fixed we also
 			// need to adjust the encryption wrapper.
 			$target = $this->fopen($targetInternalPath, 'w');
-			list(, $result) = \OC_Helper::streamCopy($source, $target);
+			[, $result] = \OC_Helper::streamCopy($source, $target);
 			if ($result and $preserveMtime) {
 				$this->touch($targetInternalPath, $sourceStorage->filemtime($sourceInternalPath));
 			}
@@ -719,6 +719,7 @@ public function getMetaData($path) {
 		$data['etag'] = $this->getETag($path);
 		$data['storage_mtime'] = $data['mtime'];
 		$data['permissions'] = $permissions;
+		$data['name'] = basename($path);
 
 		return $data;
 	}
@@ -867,4 +868,17 @@ public function writeStream(string $path, $stream, int $size = null): int {
 		}
 		return $count;
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		$dh = $this->opendir($directory);
+		if (is_resource($dh)) {
+			$basePath = rtrim($directory, '/');
+			while (($file = readdir($dh)) !== false) {
+				if (!Filesystem::isIgnoredDir($file)) {
+					$childPath = $basePath . '/' . trim($file, '/');
+					yield $this->getMetaData($childPath);
+				}
+			}
+		}
+	}
 }

lib/private/Files/Storage/Local.php

@@ -196,6 +196,7 @@ public function getMetaData($path) {
 		$data['etag'] = $this->calculateEtag($path, $stat);
 		$data['storage_mtime'] = $data['mtime'];
 		$data['permissions'] = $permissions;
+		$data['name'] = basename($path);
 
 		return $data;
 	}

lib/private/Files/Storage/Storage.php

@@ -119,4 +119,22 @@ public function releaseLock($path, $type, ILockingProvider $provider);
 	 * @throws \OCP\Lock\LockedException
 	 */
 	public function changeLock($path, $type, ILockingProvider $provider);
+
+	/**
+	 * Get the contents of a directory with metadata
+	 *
+	 * @param string $directory
+	 * @return \Traversable an iterator, containing file metadata
+	 *
+	 * The metadata array will contain the following fields
+	 *
+	 * - name
+	 * - mimetype
+	 * - mtime
+	 * - size
+	 * - etag
+	 * - storage_mtime
+	 * - permissions
+	 */
+	public function getDirectoryContent($directory): \Traversable;
 }

lib/private/Files/Storage/Wrapper/Availability.php

@@ -461,4 +461,15 @@ protected function setUnavailable(StorageNotAvailableException $e) {
 		$this->getStorageCache()->setAvailability(false, $delay);
 		throw $e;
 	}
+
+
+
+	public function getDirectoryContent($directory): \Traversable {
+		$this->checkAvailability();
+		try {
+			return parent::getDirectoryContent($directory);
+		} catch (StorageNotAvailableException $e) {
+			$this->setUnavailable($e);
+		}
+	}
 }

lib/private/Files/Storage/Wrapper/Encoding.php

@@ -534,4 +534,8 @@ public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $t
 	public function getMetaData($path) {
 		return $this->storage->getMetaData($this->findPathToUse($path));
 	}
+
+	public function getDirectoryContent($directory): \Traversable {
+		return $this->storage->getDirectoryContent($this->findPathToUse($directory));
+	}
 }