From ccb3bf05fd26919e556369d247481e5f5a46f00d Mon Sep 17 00:00:00 2001 From: Philipp Schaffrath Date: Mon, 14 Aug 2017 09:52:26 +0200 Subject: [PATCH 1/6] added LegacyStoragesService to access mount.json, added migration for external stores, marked files_external to use migrations --- .../Migrations/Version20170814051424.php | 42 ++++ apps/files_external/appinfo/info.xml | 2 +- .../Service/LegacyStoragesService.php | 222 ++++++++++++++++++ 3 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 apps/files_external/appinfo/Migrations/Version20170814051424.php create mode 100644 lib/private/Files/External/Service/LegacyStoragesService.php diff --git a/apps/files_external/appinfo/Migrations/Version20170814051424.php b/apps/files_external/appinfo/Migrations/Version20170814051424.php new file mode 100644 index 000000000000..7401790e7752 --- /dev/null +++ b/apps/files_external/appinfo/Migrations/Version20170814051424.php @@ -0,0 +1,42 @@ +query('GlobalStoragesService'); + $legacyStoragesService = new LegacyStoragesService(\OC::$server->getStoragesBackendService()); + + $legacyStorages = $legacyStoragesService->getAllStorages(); + + foreach ($legacyStorages as $legacyStorage) { + try { + $mountOptions = $legacyStorage->getMountOptions(); + if (!empty($mountOptions) && !isset($mountOptions['enable_sharing'])) { + // existing mounts must have sharing enabled by default to avoid surprises + $mountOptions['enable_sharing'] = true; + $legacyStorage->setMountOptions($mountOptions); + } + $globalStoragesService->addStorage($legacyStorage); + } catch (\Exception $exception) { + $out->warning( + 'There has been an error migrating an external storage from mount.json to the database. The affected mount point is "' . + $legacyStorage->getMountPoint() . '" and the type is "' . $legacyStorage->getBackend()->getIdentifier() . '"' + ); + } + } + } +} diff --git a/apps/files_external/appinfo/info.xml b/apps/files_external/appinfo/info.xml index a52e9d4657d8..30bd8702947d 100644 --- a/apps/files_external/appinfo/info.xml +++ b/apps/files_external/appinfo/info.xml @@ -18,7 +18,7 @@ 166048 - + true Files_External diff --git a/lib/private/Files/External/Service/LegacyStoragesService.php b/lib/private/Files/External/Service/LegacyStoragesService.php new file mode 100644 index 000000000000..efbb843c5058 --- /dev/null +++ b/lib/private/Files/External/Service/LegacyStoragesService.php @@ -0,0 +1,222 @@ + + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\Files\External\Service; +use OC\Files\External\StorageConfig; +use OCP\Files\External\IStoragesBackendService; + +/** + * Read mount config from legacy mount.json + */ +class LegacyStoragesService { + /** @var IStoragesBackendService */ + protected $backendService; + + /** + * LegacyStoragesService constructor. + * + * @param IStoragesBackendService $backendService + */ + public function __construct(IStoragesBackendService $backendService) { + $this->backendService = $backendService; + } + + + /** + * Read legacy config data + * + * @return array list of mount configs + */ + protected function readLegacyConfig() { + $mountConfig = new \OC_Mount_Config(); + return $mountConfig->readData(); + } + + /** + * Copy legacy storage options into the given storage config object. + * + * @param StorageConfig $storageConfig storage config to populate + * @param string $mountType mount type + * @param string $applicable applicable user or group + * @param array $storageOptions legacy storage options + * + * @return StorageConfig populated storage config + */ + protected function populateStorageConfigWithLegacyOptions( + &$storageConfig, + $mountType, + $applicable, + $storageOptions + ) { + $backend = $this->backendService->getBackend($storageOptions['backend']); + if (!$backend) { + throw new \UnexpectedValueException('Invalid backend ' . $storageOptions['backend']); + } + $storageConfig->setBackend($backend); + if (isset($storageOptions['authMechanism']) && $storageOptions['authMechanism'] !== 'builtin::builtin') { + $authMechanism = $this->backendService->getAuthMechanism($storageOptions['authMechanism']); + } else { + $authMechanism = $backend->getLegacyAuthMechanism($storageOptions); + $storageOptions['authMechanism'] = 'null'; // to make error handling easier + } + if (!$authMechanism) { + throw new \UnexpectedValueException('Invalid authentication mechanism ' . $storageOptions['authMechanism']); + } + $storageConfig->setAuthMechanism($authMechanism); + $storageConfig->setBackendOptions($storageOptions['options']); + if (isset($storageOptions['mountOptions'])) { + $storageConfig->setMountOptions($storageOptions['mountOptions']); + } + if (!isset($storageOptions['priority'])) { + $storageOptions['priority'] = $backend->getPriority(); + } + $storageConfig->setPriority($storageOptions['priority']); + if ($mountType === \OC_Mount_Config::MOUNT_TYPE_USER) { + $applicableUsers = $storageConfig->getApplicableUsers(); + if ($applicable !== 'all') { + $applicableUsers[] = $applicable; + $storageConfig->setApplicableUsers($applicableUsers); + } + } else if ($mountType === \OC_Mount_Config::MOUNT_TYPE_GROUP) { + $applicableGroups = $storageConfig->getApplicableGroups(); + $applicableGroups[] = $applicable; + $storageConfig->setApplicableGroups($applicableGroups); + } + return $storageConfig; + } + + /** + * Read the external storages config + * + * @return StorageConfig[] map of storage id to storage config + */ + public function getAllStorages() { + $mountPoints = $this->readLegacyConfig(); + /** + * Here is the how the horribly messy mount point array looks like + * from the mount.json file: + * + * $storageOptions = $mountPoints[$mountType][$applicable][$mountPath] + * + * - $mountType is either "user" or "group" + * - $applicable is the name of a user or group (or the current user for personal mounts) + * - $mountPath is the mount point path (where the storage must be mounted) + * - $storageOptions is a map of storage options: + * - "priority": storage priority + * - "backend": backend identifier + * - "class": LEGACY backend class name + * - "options": backend-specific options + * - "authMechanism": authentication mechanism identifier + * - "mountOptions": mount-specific options (ex: disable previews, scanner, etc) + */ + // group by storage id + /** @var StorageConfig[] $storages */ + $storages = []; + // for storages without id (legacy), group by config hash for + // later processing + $storagesWithConfigHash = []; + foreach ($mountPoints as $mountType => $applicables) { + foreach ($applicables as $applicable => $mountPaths) { + foreach ($mountPaths as $rootMountPath => $storageOptions) { + $currentStorage = null; + /** + * Flag whether the config that was read already has an id. + * If not, it will use a config hash instead and generate + * a proper id later + * + * @var boolean + */ + $hasId = false; + // the root mount point is in the format "/$user/files/the/mount/point" + // we remove the "/$user/files" prefix + $parts = explode('/', ltrim($rootMountPath, '/'), 3); + if (count($parts) < 3) { + // something went wrong, skip + \OCP\Util::writeLog( + 'files_external', + 'Could not parse mount point "' . $rootMountPath . '"', + \OCP\Util::ERROR + ); + continue; + } + $relativeMountPath = rtrim($parts[2], '/'); + // note: we cannot do this after the loop because the decrypted config + // options might be needed for the config hash + $storageOptions['options'] = \OC_Mount_Config::decryptPasswords($storageOptions['options']); + if (!isset($storageOptions['backend'])) { + $storageOptions['backend'] = $storageOptions['class']; // legacy compat + } + if (!isset($storageOptions['authMechanism'])) { + $storageOptions['authMechanism'] = null; // ensure config hash works + } + if (isset($storageOptions['id'])) { + $configId = (int)$storageOptions['id']; + if (isset($storages[$configId])) { + $currentStorage = $storages[$configId]; + } + $hasId = true; + } else { + // missing id in legacy config, need to generate + // but at this point we don't know the max-id, so use + // first group it by config hash + $storageOptions['mountpoint'] = $rootMountPath; + $configId = \OC_Mount_Config::makeConfigHash($storageOptions); + if (isset($storagesWithConfigHash[$configId])) { + $currentStorage = $storagesWithConfigHash[$configId]; + } + } + if (is_null($currentStorage)) { + // create new + $currentStorage = new StorageConfig($configId); + $currentStorage->setMountPoint($relativeMountPath); + } + try { + $this->populateStorageConfigWithLegacyOptions( + $currentStorage, + $mountType, + $applicable, + $storageOptions + ); + if ($hasId) { + $storages[$configId] = $currentStorage; + } else { + $storagesWithConfigHash[$configId] = $currentStorage; + } + } catch (\UnexpectedValueException $e) { + // dont die if a storage backend doesn't exist + \OCP\Util::writeLog( + 'files_external', + 'Could not load storage: "' . $e->getMessage() . '"', + \OCP\Util::ERROR + ); + } + } + } + } + + // convert parameter values + foreach ($storages as $storage) { + $storage->getBackend()->validateStorageDefinition($storage); + $storage->getAuthMechanism()->validateStorageDefinition($storage); + } + return $storages; + } +} \ No newline at end of file From c534d71c01d1065724d4394181f8dad3d399191c Mon Sep 17 00:00:00 2001 From: Philipp Schaffrath Date: Mon, 14 Aug 2017 10:06:18 +0200 Subject: [PATCH 2/6] added and fixed missing/outdated file header --- .../Migrations/Version20170814051424.php | 21 +++++++++++++++++++ .../Service/LegacyStoragesService.php | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/apps/files_external/appinfo/Migrations/Version20170814051424.php b/apps/files_external/appinfo/Migrations/Version20170814051424.php index 7401790e7752..ed6dbe8ee8f6 100644 --- a/apps/files_external/appinfo/Migrations/Version20170814051424.php +++ b/apps/files_external/appinfo/Migrations/Version20170814051424.php @@ -1,4 +1,25 @@ + * @author Philipp Schaffrath + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + namespace OCA\Files_External\Migrations; use OC\Files\External\Service\GlobalStoragesService; diff --git a/lib/private/Files/External/Service/LegacyStoragesService.php b/lib/private/Files/External/Service/LegacyStoragesService.php index efbb843c5058..5c272c70721a 100644 --- a/lib/private/Files/External/Service/LegacyStoragesService.php +++ b/lib/private/Files/External/Service/LegacyStoragesService.php @@ -1,8 +1,9 @@ + * @author Philipp Schaffrath * - * @copyright Copyright (c) 2016, ownCloud, Inc. + * @copyright Copyright (c) 2017, ownCloud GmbH * @license AGPL-3.0 * * This code is free software: you can redistribute it and/or modify From 9f7c1856416aec158391e38c7baf1faddaed6cfb Mon Sep 17 00:00:00 2001 From: Philipp Schaffrath Date: Mon, 14 Aug 2017 10:07:57 +0200 Subject: [PATCH 3/6] fixed indentation issues --- .../appinfo/Migrations/Version20170814051424.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/files_external/appinfo/Migrations/Version20170814051424.php b/apps/files_external/appinfo/Migrations/Version20170814051424.php index ed6dbe8ee8f6..ae91ac51fe15 100644 --- a/apps/files_external/appinfo/Migrations/Version20170814051424.php +++ b/apps/files_external/appinfo/Migrations/Version20170814051424.php @@ -32,16 +32,16 @@ */ class Version20170814051424 implements ISimpleMigration { - /** - * @param IOutput $out - */ - public function run(IOutput $out) { + /** + * @param IOutput $out + */ + public function run(IOutput $out) { /** @var GlobalStoragesService $globalStoragesService */ - $globalStoragesService = \OC::$server->query('GlobalStoragesService'); - $legacyStoragesService = new LegacyStoragesService(\OC::$server->getStoragesBackendService()); + $globalStoragesService = \OC::$server->query('GlobalStoragesService'); + $legacyStoragesService = new LegacyStoragesService(\OC::$server->getStoragesBackendService()); - $legacyStorages = $legacyStoragesService->getAllStorages(); + $legacyStorages = $legacyStoragesService->getAllStorages(); foreach ($legacyStorages as $legacyStorage) { try { @@ -59,5 +59,5 @@ public function run(IOutput $out) { ); } } - } + } } From 0630c38105cf2b6ba5176159c1f00ea76228fe3b Mon Sep 17 00:00:00 2001 From: Philipp Schaffrath Date: Mon, 14 Aug 2017 11:02:47 +0200 Subject: [PATCH 4/6] using LegacyUtil to not depend on OC_Mount_Config if possible, re introduced makeConfigHash, fixed some minor things --- lib/private/Files/External/LegacyUtil.php | 22 +++++++++++ .../Service/LegacyStoragesService.php | 38 ++++++++++--------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/lib/private/Files/External/LegacyUtil.php b/lib/private/Files/External/LegacyUtil.php index 9ff560be44ac..dc460eea54f2 100644 --- a/lib/private/Files/External/LegacyUtil.php +++ b/lib/private/Files/External/LegacyUtil.php @@ -308,4 +308,26 @@ private static function getCipher() { $cipher->setKey(\OC::$server->getConfig()->getSystemValue('passwordsalt', null)); return $cipher; } + + /** + * Computes a hash based on the given configuration. + * This is mostly used to find out whether configurations + * are the same. + * + * @param array $config + * @return string + */ + public static function makeConfigHash($config) { + $data = json_encode( + array( + 'c' => $config['backend'], + 'a' => $config['authMechanism'], + 'm' => $config['mountpoint'], + 'o' => $config['options'], + 'p' => isset($config['priority']) ? $config['priority'] : -1, + 'mo' => isset($config['mountOptions']) ? $config['mountOptions'] : [], + ) + ); + return hash('md5', $data); + } } diff --git a/lib/private/Files/External/Service/LegacyStoragesService.php b/lib/private/Files/External/Service/LegacyStoragesService.php index 5c272c70721a..f3faddf6e609 100644 --- a/lib/private/Files/External/Service/LegacyStoragesService.php +++ b/lib/private/Files/External/Service/LegacyStoragesService.php @@ -17,38 +17,41 @@ * * You should have received a copy of the GNU Affero General Public License, version 3, * along with this program. If not, see - * */ namespace OC\Files\External\Service; +use OC\Files\External\LegacyUtil; use OC\Files\External\StorageConfig; use OCP\Files\External\IStoragesBackendService; +use OCP\Util; /** * Read mount config from legacy mount.json */ class LegacyStoragesService { - /** @var IStoragesBackendService */ + /** + * @var IStoragesBackendService + */ protected $backendService; /** - * LegacyStoragesService constructor. - * + * @var LegacyUtil + */ + private $legacyUtil; + + /** * @param IStoragesBackendService $backendService */ public function __construct(IStoragesBackendService $backendService) { $this->backendService = $backendService; + $this->legacyUtil = new LegacyUtil(); } - /** - * Read legacy config data - * * @return array list of mount configs */ protected function readLegacyConfig() { - $mountConfig = new \OC_Mount_Config(); - return $mountConfig->readData(); + return $this->legacyUtil->readData(); } /** @@ -90,13 +93,14 @@ protected function populateStorageConfigWithLegacyOptions( $storageOptions['priority'] = $backend->getPriority(); } $storageConfig->setPriority($storageOptions['priority']); - if ($mountType === \OC_Mount_Config::MOUNT_TYPE_USER) { + + if ($mountType === LegacyUtil::MOUNT_TYPE_USER) { $applicableUsers = $storageConfig->getApplicableUsers(); if ($applicable !== 'all') { $applicableUsers[] = $applicable; $storageConfig->setApplicableUsers($applicableUsers); } - } else if ($mountType === \OC_Mount_Config::MOUNT_TYPE_GROUP) { + } else if ($mountType === LegacyUtil::MOUNT_TYPE_GROUP) { $applicableGroups = $storageConfig->getApplicableGroups(); $applicableGroups[] = $applicable; $storageConfig->setApplicableGroups($applicableGroups); @@ -151,10 +155,10 @@ public function getAllStorages() { $parts = explode('/', ltrim($rootMountPath, '/'), 3); if (count($parts) < 3) { // something went wrong, skip - \OCP\Util::writeLog( + Util::writeLog( 'files_external', 'Could not parse mount point "' . $rootMountPath . '"', - \OCP\Util::ERROR + Util::ERROR ); continue; } @@ -179,7 +183,7 @@ public function getAllStorages() { // but at this point we don't know the max-id, so use // first group it by config hash $storageOptions['mountpoint'] = $rootMountPath; - $configId = \OC_Mount_Config::makeConfigHash($storageOptions); + $configId = LegacyUtil::makeConfigHash($storageOptions); if (isset($storagesWithConfigHash[$configId])) { $currentStorage = $storagesWithConfigHash[$configId]; } @@ -202,11 +206,11 @@ public function getAllStorages() { $storagesWithConfigHash[$configId] = $currentStorage; } } catch (\UnexpectedValueException $e) { - // dont die if a storage backend doesn't exist - \OCP\Util::writeLog( + // don't die if a storage backend doesn't exist + Util::writeLog( 'files_external', 'Could not load storage: "' . $e->getMessage() . '"', - \OCP\Util::ERROR + Util::ERROR ); } } From b0571b861fd70da002d22d3621c0ac3ac846b331 Mon Sep 17 00:00:00 2001 From: Philipp Schaffrath Date: Mon, 14 Aug 2017 11:46:17 +0200 Subject: [PATCH 5/6] use the first available auth mechanism as fallback --- lib/private/Files/External/Service/LegacyStoragesService.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/private/Files/External/Service/LegacyStoragesService.php b/lib/private/Files/External/Service/LegacyStoragesService.php index f3faddf6e609..06b472bea58f 100644 --- a/lib/private/Files/External/Service/LegacyStoragesService.php +++ b/lib/private/Files/External/Service/LegacyStoragesService.php @@ -78,7 +78,8 @@ protected function populateStorageConfigWithLegacyOptions( if (isset($storageOptions['authMechanism']) && $storageOptions['authMechanism'] !== 'builtin::builtin') { $authMechanism = $this->backendService->getAuthMechanism($storageOptions['authMechanism']); } else { - $authMechanism = $backend->getLegacyAuthMechanism($storageOptions); + $authMechanisms = $this->backendService->getAuthMechanisms(); + $authMechanism = $authMechanisms[0]; $storageOptions['authMechanism'] = 'null'; // to make error handling easier } if (!$authMechanism) { From e27abbdcaf7a680ccfa99680cd7d415ad8fc0a04 Mon Sep 17 00:00:00 2001 From: Philipp Schaffrath Date: Mon, 14 Aug 2017 12:17:25 +0200 Subject: [PATCH 6/6] increased version, so the migrations run --- apps/files_external/appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/files_external/appinfo/info.xml b/apps/files_external/appinfo/info.xml index 30bd8702947d..899489671137 100644 --- a/apps/files_external/appinfo/info.xml +++ b/apps/files_external/appinfo/info.xml @@ -13,7 +13,7 @@ admin-external-storage false - 0.7.0 + 0.7.1