diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index 1f878df1564eb..c21b65017a8c3 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -34,6 +34,7 @@ namespace OCA\DAV\Connector\Sabre; use OC\Files\Filesystem; +use OC\Files\Storage\Storage; use OCA\DAV\Connector\Sabre\Exception\EntityTooLarge; use OCA\DAV\Connector\Sabre\Exception\FileLocked; use OCA\DAV\Connector\Sabre\Exception\Forbidden as DAVForbiddenException; @@ -133,6 +134,10 @@ public function put($data) { list($count, $result) = \OC_Helper::streamCopy($data, $target); fclose($target); + if (!self::isChecksumValid($partStorage, $internalPartPath)) { + throw new BadRequest('The computed checksum does not match the one received from the client.'); + } + if ($result === false) { $expected = -1; if (isset($_SERVER['CONTENT_LENGTH'])) { @@ -222,15 +227,17 @@ public function put($data) { $this->refreshInfo(); - if (isset($request->server['HTTP_OC_CHECKSUM'])) { - $checksum = trim($request->server['HTTP_OC_CHECKSUM']); - $this->fileView->putFileInfo($this->path, ['checksum' => $checksum]); - $this->refreshInfo(); - } else if ($this->getChecksum() !== null && $this->getChecksum() !== '') { - $this->fileView->putFileInfo($this->path, ['checksum' => '']); - $this->refreshInfo(); + $meta = $partStorage->getCache()->get($internalPartPath); + + if (isset($meta['checksum'])) { + $this->fileView->putFileInfo( + $this->path, + ['checksum' => $meta['checksum']] + ); } + $this->refreshInfo(); + } catch (StorageNotAvailableException $e) { throw new ServiceUnavailable("Failed to check file size: " . $e->getMessage()); } @@ -441,6 +448,10 @@ private function createFileChunked($data) { $chunk_handler->file_assemble($partStorage, $partInternalPath); + if (!self::isChecksumValid($partStorage, $partInternalPath)) { + throw new BadRequest('The computed checksum does not match the one received from the client.'); + } + // here is the final atomic rename $renameOkay = $targetStorage->moveFromStorage($partStorage, $partInternalPath, $targetInternalPath); $fileExists = $targetStorage->file_exists($targetInternalPath); @@ -479,13 +490,20 @@ private function createFileChunked($data) { // FIXME: should call refreshInfo but can't because $this->path is not the of the final file $info = $this->fileView->getFileInfo($targetPath); - if (isset($request->server['HTTP_OC_CHECKSUM'])) { - $checksum = trim($request->server['HTTP_OC_CHECKSUM']); - $this->fileView->putFileInfo($targetPath, ['checksum' => $checksum]); - } else if ($info->getChecksum() !== null && $info->getChecksum() !== '') { - $this->fileView->putFileInfo($this->path, ['checksum' => '']); + + if (isset($partStorage) && isset($partInternalPath)) { + $checksums = $partStorage->getCache()->get($partInternalPath)['checksum']; + } else { + $checksums = $targetStorage->getCache()->get($targetInternalPath)['checksum']; } + $this->fileView->putFileInfo( + $targetPath, + ['checksum' => $checksums] + ); + + $this->refreshInfo(); + $this->fileView->unlockFile($targetPath, ILockingProvider::LOCK_SHARED); return $info->getEtag(); @@ -500,6 +518,29 @@ private function createFileChunked($data) { return null; } + /** + * will return true if checksum was not provided in request + * + * @param Storage $storage + * @param $path + * @return bool + */ + private static function isChecksumValid(Storage $storage, $path) { + $meta = $storage->getMetaData($path); + $request = \OC::$server->getRequest(); + + if (!isset($request->server['HTTP_OC_CHECKSUM']) || !isset($meta['checksum'])) { + // No comparison possible, skip the check + return true; + } + + $expectedChecksum = trim($request->server['HTTP_OC_CHECKSUM']); + $computedChecksums = $meta['checksum']; + + return strpos($computedChecksums, $expectedChecksum) !== false; + + } + /** * Returns whether a part file is needed for the given storage * or whether the file can be assembled/uploaded directly on the @@ -562,12 +603,30 @@ private function convertToSabreException(\Exception $e) { throw new \Sabre\DAV\Exception($e->getMessage(), 0, $e); } + /** - * Get the checksum for this file - * + * Set $algo to get a specific checksum, leave null to get all checksums + * (space seperated) + * @param null $algo * @return string */ - public function getChecksum() { - return $this->info->getChecksum(); + public function getChecksum($algo = null) { + $allChecksums = $this->info->getChecksum(); + + if (!$algo) { + return $allChecksums; + } + + $checksums = explode(' ', $allChecksums); + $algoPrefix = strtoupper($algo) . ':'; + + foreach ($checksums as $checksum) { + // starts with $algoPrefix + if (substr($checksum, 0, strlen($algoPrefix)) === $algoPrefix) { + return $checksum; + } + } + + return ''; } } diff --git a/apps/dav/lib/Connector/Sabre/FilesPlugin.php b/apps/dav/lib/Connector/Sabre/FilesPlugin.php index 1b1f43df9eac4..b240d62304ba1 100644 --- a/apps/dav/lib/Connector/Sabre/FilesPlugin.php +++ b/apps/dav/lib/Connector/Sabre/FilesPlugin.php @@ -265,7 +265,7 @@ function httpGet(RequestInterface $request, ResponseInterface $response) { if ($node instanceof \OCA\DAV\Connector\Sabre\File) { //Add OC-Checksum header /** @var $node File */ - $checksum = $node->getChecksum(); + $checksum = $node->getChecksum('sha1'); if ($checksum !== null && $checksum !== '') { $response->addHeader('OC-Checksum', $checksum); } @@ -360,7 +360,7 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node) }); $propFind->handle(self::CHECKSUMS_PROPERTYNAME, function() use ($node) { - $checksum = $node->getChecksum(); + $checksum = $node->getChecksum('sha1'); if ($checksum === NULL || $checksum === '') { return null; } diff --git a/apps/files/lib/Capabilities.php b/apps/files/lib/Capabilities.php index 2b6bf57b90dd6..76919f8aa2293 100644 --- a/apps/files/lib/Capabilities.php +++ b/apps/files/lib/Capabilities.php @@ -53,6 +53,10 @@ public function __construct(IConfig $config) { */ public function getCapabilities() { return [ + 'checksums' => [ + 'supportedTypes' => ['SHA1'], + 'preferredUploadType' => 'SHA1' + ], 'files' => [ 'bigfilechunking' => true, 'blacklisted_files' => $this->config->getSystemValue('blacklisted_files', ['.htaccess']), diff --git a/apps/files_trashbin/tests/TrashbinTest.php b/apps/files_trashbin/tests/TrashbinTest.php index 7e4cdb112e86d..fe3c77944143c 100644 --- a/apps/files_trashbin/tests/TrashbinTest.php +++ b/apps/files_trashbin/tests/TrashbinTest.php @@ -147,50 +147,8 @@ protected function tearDown() { parent::tearDown(); } - /** - * test expiration of files older then the max storage time defined for the trash - */ - public function testExpireOldFiles() { - - $currentTime = time(); - $expireAt = $currentTime - 2 * 24 * 60 * 60; - $expiredDate = $currentTime - 3 * 24 * 60 * 60; - - // create some files - \OC\Files\Filesystem::file_put_contents('file1.txt', 'file1'); - \OC\Files\Filesystem::file_put_contents('file2.txt', 'file2'); - \OC\Files\Filesystem::file_put_contents('file3.txt', 'file3'); - - // delete them so that they end up in the trash bin - \OC\Files\Filesystem::unlink('file1.txt'); - \OC\Files\Filesystem::unlink('file2.txt'); - \OC\Files\Filesystem::unlink('file3.txt'); - - //make sure that files are in the trash bin - $filesInTrash = OCA\Files_Trashbin\Helper::getTrashFiles('/', self::TEST_TRASHBIN_USER1, 'name'); - $this->assertSame(3, count($filesInTrash)); - // every second file will get a date in the past so that it will get expired - $manipulatedList = $this->manipulateDeleteTime($filesInTrash, $this->trashRoot1, $expiredDate); - - $testClass = new TrashbinForTesting(); - list($sizeOfDeletedFiles, $count) = $testClass->dummyDeleteExpiredFiles($manipulatedList, $expireAt); - - $this->assertSame(10, $sizeOfDeletedFiles); - $this->assertSame(2, $count); - - // only file2.txt should be left - $remainingFiles = array_slice($manipulatedList, $count); - $this->assertSame(1, count($remainingFiles)); - $remainingFile = reset($remainingFiles); - $this->assertSame('file2.txt', $remainingFile['name']); - // check that file1.txt and file3.txt was really deleted - $newTrashContent = OCA\Files_Trashbin\Helper::getTrashFiles('/', self::TEST_TRASHBIN_USER1); - $this->assertSame(1, count($newTrashContent)); - $element = reset($newTrashContent); - $this->assertSame('file2.txt', $element['name']); - } /** * test expiration of files older then the max storage time defined for the trash @@ -268,6 +226,52 @@ public function testExpireOldFilesShared() { $this->verifyArray($filesInTrashUser1AfterDelete, array('user1-2.txt', 'user1-4.txt')); } + /** + * test expiration of files older then the max storage time defined for the trash + */ + public function testExpireOldFiles() { + + $currentTime = time(); + $expireAt = $currentTime - 2 * 24 * 60 * 60; + $expiredDate = $currentTime - 3 * 24 * 60 * 60; + + // create some files + \OC\Files\Filesystem::file_put_contents('file1.txt', 'file1'); + \OC\Files\Filesystem::file_put_contents('file2.txt', 'file2'); + \OC\Files\Filesystem::file_put_contents('file3.txt', 'file3'); + + // delete them so that they end up in the trash bin + \OC\Files\Filesystem::unlink('file1.txt'); + \OC\Files\Filesystem::unlink('file2.txt'); + \OC\Files\Filesystem::unlink('file3.txt'); + + //make sure that files are in the trash bin + $filesInTrash = OCA\Files_Trashbin\Helper::getTrashFiles('/', self::TEST_TRASHBIN_USER1, 'name'); + $this->assertSame(3, count($filesInTrash)); + + // every second file will get a date in the past so that it will get expired + $manipulatedList = $this->manipulateDeleteTime($filesInTrash, $this->trashRoot1, $expiredDate); + + $testClass = new TrashbinForTesting(); + list($sizeOfDeletedFiles, $count) = $testClass->dummyDeleteExpiredFiles($manipulatedList, $expireAt); + + $this->assertSame(10, $sizeOfDeletedFiles); + $this->assertSame(2, $count); + + // only file2.txt should be left + $remainingFiles = array_slice($manipulatedList, $count); + $this->assertSame(1, count($remainingFiles)); + $remainingFile = reset($remainingFiles); + $this->assertSame('file2.txt', $remainingFile['name']); + + // check that file1.txt and file3.txt was really deleted + $newTrashContent = OCA\Files_Trashbin\Helper::getTrashFiles('/', self::TEST_TRASHBIN_USER1); + $this->assertSame(1, count($newTrashContent)); + $element = reset($newTrashContent); + $this->assertSame('file2.txt', $element['name']); + } + + /** * verify that the array contains the expected results * diff --git a/apps/files_versions/tests/VersioningTest.php b/apps/files_versions/tests/VersioningTest.php index a1649b2b600d0..2834236fe9e13 100644 --- a/apps/files_versions/tests/VersioningTest.php +++ b/apps/files_versions/tests/VersioningTest.php @@ -119,6 +119,57 @@ protected function tearDown() { parent::tearDown(); } + + public function testMoveFileIntoSharedFolderAsRecipient() { + + \OC\Files\Filesystem::mkdir('folder1'); + $fileInfo = \OC\Files\Filesystem::getFileInfo('folder1'); + + $node = \OC::$server->getUserFolder(self::TEST_VERSIONS_USER)->get('folder1'); + $share = \OC::$server->getShareManager()->newShare(); + $share->setNode($node) + ->setShareType(\OCP\Share::SHARE_TYPE_USER) + ->setSharedBy(self::TEST_VERSIONS_USER) + ->setSharedWith(self::TEST_VERSIONS_USER2) + ->setPermissions(\OCP\Constants::PERMISSION_ALL); + $share = \OC::$server->getShareManager()->createShare($share); + + self::loginHelper(self::TEST_VERSIONS_USER2); + $versionsFolder2 = '/' . self::TEST_VERSIONS_USER2 . '/files_versions'; + \OC\Files\Filesystem::file_put_contents('test.txt', 'test file'); + + $t1 = time(); + // second version is two weeks older, this way we make sure that no + // version will be expired + $t2 = $t1 - 60 * 60 * 24 * 14; + + $this->rootView->mkdir($versionsFolder2); + // create some versions + $v1 = $versionsFolder2 . '/test.txt.v' . $t1; + $v2 = $versionsFolder2 . '/test.txt.v' . $t2; + + $this->rootView->file_put_contents($v1, 'version1'); + $this->rootView->file_put_contents($v2, 'version2'); + + // move file into the shared folder as recipient + \OC\Files\Filesystem::rename('/test.txt', '/folder1/test.txt'); + + $this->assertFalse($this->rootView->file_exists($v1)); + $this->assertFalse($this->rootView->file_exists($v2)); + + self::loginHelper(self::TEST_VERSIONS_USER); + + $versionsFolder1 = '/' . self::TEST_VERSIONS_USER . '/files_versions'; + + $v1Renamed = $versionsFolder1 . '/folder1/test.txt.v' . $t1; + $v2Renamed = $versionsFolder1 . '/folder1/test.txt.v' . $t2; + + $this->assertTrue($this->rootView->file_exists($v1Renamed)); + $this->assertTrue($this->rootView->file_exists($v2Renamed)); + + \OC::$server->getShareManager()->deleteShare($share); + } + /** * @medium * test expire logic @@ -380,56 +431,6 @@ public function testMoveFolder() { } - public function testMoveFileIntoSharedFolderAsRecipient() { - - \OC\Files\Filesystem::mkdir('folder1'); - $fileInfo = \OC\Files\Filesystem::getFileInfo('folder1'); - - $node = \OC::$server->getUserFolder(self::TEST_VERSIONS_USER)->get('folder1'); - $share = \OC::$server->getShareManager()->newShare(); - $share->setNode($node) - ->setShareType(\OCP\Share::SHARE_TYPE_USER) - ->setSharedBy(self::TEST_VERSIONS_USER) - ->setSharedWith(self::TEST_VERSIONS_USER2) - ->setPermissions(\OCP\Constants::PERMISSION_ALL); - $share = \OC::$server->getShareManager()->createShare($share); - - self::loginHelper(self::TEST_VERSIONS_USER2); - $versionsFolder2 = '/' . self::TEST_VERSIONS_USER2 . '/files_versions'; - \OC\Files\Filesystem::file_put_contents('test.txt', 'test file'); - - $t1 = time(); - // second version is two weeks older, this way we make sure that no - // version will be expired - $t2 = $t1 - 60 * 60 * 24 * 14; - - $this->rootView->mkdir($versionsFolder2); - // create some versions - $v1 = $versionsFolder2 . '/test.txt.v' . $t1; - $v2 = $versionsFolder2 . '/test.txt.v' . $t2; - - $this->rootView->file_put_contents($v1, 'version1'); - $this->rootView->file_put_contents($v2, 'version2'); - - // move file into the shared folder as recipient - \OC\Files\Filesystem::rename('/test.txt', '/folder1/test.txt'); - - $this->assertFalse($this->rootView->file_exists($v1)); - $this->assertFalse($this->rootView->file_exists($v2)); - - self::loginHelper(self::TEST_VERSIONS_USER); - - $versionsFolder1 = '/' . self::TEST_VERSIONS_USER . '/files_versions'; - - $v1Renamed = $versionsFolder1 . '/folder1/test.txt.v' . $t1; - $v2Renamed = $versionsFolder1 . '/folder1/test.txt.v' . $t2; - - $this->assertTrue($this->rootView->file_exists($v1Renamed)); - $this->assertTrue($this->rootView->file_exists($v2Renamed)); - - \OC::$server->getShareManager()->deleteShare($share); - } - public function testMoveFolderIntoSharedFolderAsRecipient() { \OC\Files\Filesystem::mkdir('folder1'); diff --git a/build/integration/config/behat.yml b/build/integration/config/behat.yml index 3573f9d6a6b93..7882966b20a45 100644 --- a/build/integration/config/behat.yml +++ b/build/integration/config/behat.yml @@ -20,8 +20,6 @@ default: baseUrl: http://localhost:8080 - CalDavContext: baseUrl: http://localhost:8080 - - ChecksumsContext: - baseUrl: http://localhost:8080 - CommandLineContext: baseUrl: http://localhost:8080 ocPath: ../../ diff --git a/build/integration/features/bootstrap/BasicStructure.php b/build/integration/features/bootstrap/BasicStructure.php index 9e7aa272d0aff..aeaacb932e5ec 100644 --- a/build/integration/features/bootstrap/BasicStructure.php +++ b/build/integration/features/bootstrap/BasicStructure.php @@ -35,6 +35,7 @@ trait BasicStructure { use Auth; + use Checksums; use Trashbin; /** @var string */ diff --git a/build/integration/features/bootstrap/Checksums.php b/build/integration/features/bootstrap/Checksums.php new file mode 100644 index 0000000000000..6495b58443a54 --- /dev/null +++ b/build/integration/features/bootstrap/Checksums.php @@ -0,0 +1,185 @@ +adminUser; + } else { + return $this->regularUser; + } + } + + /** + * @When user :user uploads file :source to :destination with checksum :checksum + * @param string $user + * @param string $source + * @param string $destination + * @param string $checksum + */ + public function userUploadsFileToWithChecksum($user, $source, $destination, $checksum) { + $file = \GuzzleHttp\Stream\Stream::factory(fopen($source, 'r')); + $this->response = $this->makeDavRequest($user, + 'PUT', + $destination, + ['OC-Checksum' => $checksum], + $file, + "files"); + } + + /** + * @Then The webdav response should have a status code :statusCode + * @param int $statusCode + * @throws \Exception + */ + public function theWebdavResponseShouldHaveAStatusCode($statusCode) { + if((int)$statusCode !== $this->response->getStatusCode()) { + throw new \Exception("Expected $statusCode, got ".$this->response->getStatusCode()); + } + } + + /** + * @When user :user request the checksum of :path via propfind + * @param string $user + * @param string $path + */ + public function userRequestTheChecksumOfViaPropfind($user, $path) { + $client = new Client(); + $request = $client->createRequest( + 'PROPFIND', + substr($this->baseUrl, 0, -4) . $this->getDavFilesPath($user) . $path, + [ + 'body' => ' + + + + +', + 'auth' => [ + $user, + $this->getPasswordForUser($user), + ] + ] + ); + $this->response = $client->send($request); + } + + /** + * @Then The webdav checksum should match :checksum + * @param string $checksum + * @throws \Exception + */ + public function theWebdavChecksumShouldMatch($checksum) { + $service = new Sabre\DAV\Xml\Service(); + /** @var MultiStatus $parsed */ + $parsed = $service->parse($this->response->getBody()->getContents()); + + $props = $parsed->getResponses()[0]->getResponseProperties(); + if (isset($props[200]) && isset($props[200]['{http://owncloud.org/ns}checksums'])) { + $checksums = $props[200]['{http://owncloud.org/ns}checksums'][0]; + if ($checksums['value'] !== $checksum) { + throw new \Exception("Expected $checksum, got " . $checksums['value']); + } + } else { + throw new \Exception('No checksum provided'); + } + } + + /** + * @Then The header checksum should match :checksum + * @param string $checksum + * @throws \Exception + */ + public function theHeaderChecksumShouldMatch($checksum) + { + if ($this->response->getHeader('OC-Checksum') !== $checksum) { + throw new \Exception("Expected $checksum, got ".$this->response->getHeader('OC-Checksum')); + } + } + + /** + * @Given User :user copied file :source to :destination + * @param string $user + * @param string $source + * @param string $destination + */ + public function userCopiedFileTo($user, $source, $destination) { + $client = new Client(); + $request = $client->createRequest( + 'COPY', + substr($this->baseUrl, 0, -4) . $this->davPath . $source, + [ + 'auth' => [ + $user, + $this->getPasswordForUser($user), + ], + 'headers' => [ + 'Destination' => substr($this->baseUrl, 0, -4) . $this->davPath . $destination, + ], + ] + ); + $this->response = $client->send($request); + } + + /** + * @Then The webdav checksum should be empty + */ + public function theWebdavChecksumShouldBeEmpty() + { + $service = new Sabre\Xml\Service(); + $parsed = $service->parse($this->response->getBody()->getContents()); + + /* + * Fetch the checksum array + * Maybe we want to do this a bit cleaner ;) + */ + $status = $parsed[0]['value'][1]['value'][1]['value']; + + if ($status !== 'HTTP/1.1 404 Not Found') { + throw new \Exception("Expected 'HTTP/1.1 404 Not Found', got ".$status); + } + } + + /** + * @Then The OC-Checksum header should not be there + */ + public function theOcChecksumHeaderShouldNotBeThere() + { + if ($this->response->hasHeader('OC-Checksum')) { + throw new \Exception("Expected no checksum header but got ".$this->response->getHeader('OC-Checksum')); + } + } + + /** + * @Given user :user uploads chunk file :num of :total with :data to :destination with checksum :checksum + * @param string $user + * @param int $num + * @param int $total + * @param string $data + * @param string $destination + * @param string $checksum + */ + public function userUploadsChunkFileOfWithToWithChecksum($user, $num, $total, $data, $destination, $checksum) { + $num -= 1; + $data = \GuzzleHttp\Stream\Stream::factory($data); + $file = $destination . '-chunking-42-' . $total . '-' . $num; + $this->response = $this->makeDavRequest($user, + 'PUT', + $file, + ['OC-Checksum' => $checksum, 'OC-Chunked' => '1'], + $data, + "files"); + } +} diff --git a/build/integration/features/bootstrap/ChecksumsContext.php b/build/integration/features/bootstrap/ChecksumsContext.php deleted file mode 100644 index 4dd43db852f65..0000000000000 --- a/build/integration/features/bootstrap/ChecksumsContext.php +++ /dev/null @@ -1,270 +0,0 @@ - - * @author Roeland Jago Douma - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ -require __DIR__ . '/../../vendor/autoload.php'; - -use GuzzleHttp\Client; -use GuzzleHttp\Message\ResponseInterface; - -class ChecksumsContext implements \Behat\Behat\Context\Context { - /** @var string */ - private $baseUrl; - /** @var Client */ - private $client; - /** @var ResponseInterface */ - private $response; - - /** - * @param string $baseUrl - */ - public function __construct($baseUrl) { - $this->baseUrl = $baseUrl; - - // in case of ci deployment we take the server url from the environment - $testServerUrl = getenv('TEST_SERVER_URL'); - if ($testServerUrl !== false) { - $this->baseUrl = substr($testServerUrl, 0, -5); - } - } - - /** @BeforeScenario */ - public function setUpScenario() { - $this->client = new Client(); - } - - /** @AfterScenario */ - public function tearDownScenario() { - } - - - /** - * @param string $userName - * @return string - */ - private function getPasswordForUser($userName) { - if($userName === 'admin') { - return 'admin'; - } - return '123456'; - } - - /** - * @When user :user uploads file :source to :destination with checksum :checksum - * @param string $user - * @param string $source - * @param string $destination - * @param string $checksum - */ - public function userUploadsFileToWithChecksum($user, $source, $destination, $checksum) - { - $file = \GuzzleHttp\Stream\Stream::factory(fopen($source, 'r')); - try { - $this->response = $this->client->put( - $this->baseUrl . '/remote.php/webdav' . $destination, - [ - 'auth' => [ - $user, - $this->getPasswordForUser($user) - ], - 'body' => $file, - 'headers' => [ - 'OC-Checksum' => $checksum - ] - ] - ); - } catch (\GuzzleHttp\Exception\ServerException $e) { - // 4xx and 5xx responses cause an exception - $this->response = $e->getResponse(); - } - } - - /** - * @Then The webdav response should have a status code :statusCode - * @param int $statusCode - * @throws \Exception - */ - public function theWebdavResponseShouldHaveAStatusCode($statusCode) { - if((int)$statusCode !== $this->response->getStatusCode()) { - throw new \Exception("Expected $statusCode, got ".$this->response->getStatusCode()); - } - } - - /** - * @When user :user request the checksum of :path via propfind - * @param string $user - * @param string $path - */ - public function userRequestTheChecksumOfViaPropfind($user, $path) - { - $request = $this->client->createRequest( - 'PROPFIND', - $this->baseUrl . '/remote.php/webdav' . $path, - [ - 'body' => ' - - - - -', - 'auth' => [ - $user, - $this->getPasswordForUser($user), - ] - ] - ); - $this->response = $this->client->send($request); - } - - /** - * @Then The webdav checksum should match :checksum - * @param string $checksum - * @throws \Exception - */ - public function theWebdavChecksumShouldMatch($checksum) - { - $service = new Sabre\Xml\Service(); - $parsed = $service->parse($this->response->getBody()->getContents()); - - /* - * Fetch the checksum array - * Maybe we want to do this a bit cleaner ;) - */ - $checksums = $parsed[0]['value'][1]['value'][0]['value'][0]; - - if ($checksums['value'][0]['value'] !== $checksum) { - throw new \Exception("Expected $checksum, got ".$checksums['value'][0]['value']); - } - } - - /** - * @When user :user downloads the file :path - * @param string $user - * @param string $path - */ - public function userDownloadsTheFile($user, $path) - { - $this->response = $this->client->get( - $this->baseUrl . '/remote.php/webdav' . $path, - [ - 'auth' => [ - $user, - $this->getPasswordForUser($user), - ] - ] - ); - } - - /** - * @Then The header checksum should match :checksum - * @param string $checksum - * @throws \Exception - */ - public function theHeaderChecksumShouldMatch($checksum) - { - if ($this->response->getHeader('OC-Checksum') !== $checksum) { - throw new \Exception("Expected $checksum, got ".$this->response->getHeader('OC-Checksum')); - } - } - - /** - * @Given User :user copied file :source to :destination - * @param string $user - * @param string $source - * @param string $destination - */ - public function userCopiedFileTo($user, $source, $destination) - { - $request = $this->client->createRequest( - 'MOVE', - $this->baseUrl . '/remote.php/webdav' . $source, - [ - 'auth' => [ - $user, - $this->getPasswordForUser($user), - ], - 'headers' => [ - 'Destination' => $this->baseUrl . '/remote.php/webdav' . $destination, - ], - ] - ); - $this->response = $this->client->send($request); - } - - /** - * @Then The webdav checksum should be empty - */ - public function theWebdavChecksumShouldBeEmpty() - { - $service = new Sabre\Xml\Service(); - $parsed = $service->parse($this->response->getBody()->getContents()); - - /* - * Fetch the checksum array - * Maybe we want to do this a bit cleaner ;) - */ - $status = $parsed[0]['value'][1]['value'][1]['value']; - - if ($status !== 'HTTP/1.1 404 Not Found') { - throw new \Exception("Expected 'HTTP/1.1 404 Not Found', got ".$status); - } - } - - /** - * @Then The OC-Checksum header should not be there - */ - public function theOcChecksumHeaderShouldNotBeThere() - { - if ($this->response->hasHeader('OC-Checksum')) { - throw new \Exception("Expected no checksum header but got ".$this->response->getHeader('OC-Checksum')); - } - } - - /** - * @Given user :user uploads chunk file :num of :total with :data to :destination with checksum :checksum - * @param string $user - * @param int $num - * @param int $total - * @param string $data - * @param string $destination - * @param string $checksum - */ - public function userUploadsChunkFileOfWithToWithChecksum($user, $num, $total, $data, $destination, $checksum) - { - $num -= 1; - $this->response = $this->client->put( - $this->baseUrl . '/remote.php/webdav' . $destination . '-chunking-42-'.$total.'-'.$num, - [ - 'auth' => [ - $user, - $this->getPasswordForUser($user) - ], - 'body' => $data, - 'headers' => [ - 'OC-Checksum' => $checksum, - 'OC-Chunked' => '1', - ] - ] - ); - - } -} diff --git a/build/integration/features/bootstrap/WebDav.php b/build/integration/features/bootstrap/WebDav.php index 3a018a2d0faca..4a908de2155c0 100644 --- a/build/integration/features/bootstrap/WebDav.php +++ b/build/integration/features/bootstrap/WebDav.php @@ -84,7 +84,7 @@ public function makeDavRequest($user, $method, $path, $headers, $body = null, $t $fullUrl = substr($this->baseUrl, 0, -4) . $this->getDavFilesPath($user) . "$path"; } else if ( $type === "uploads" ){ $fullUrl = substr($this->baseUrl, 0, -4) . $this->davPath . "$path"; - } + } $client = new GClient(); $options = []; if ($user === 'admin') { @@ -230,6 +230,19 @@ public function downloadingFile($fileName) { } } + /** + * @When user :user downloads the file :fileName + * @param string $user + * @param string $fileName + */ + public function userDownloadsTheFile($user, $fileName) { + try { + $this->response = $this->makeDavRequest($user, 'GET', $fileName, []); + } catch (\GuzzleHttp\Exception\ClientException $e) { + $this->response = $e->getResponse(); + } + } + /** * @Then The following headers should be set * @param \Behat\Gherkin\Node\TableNode $table @@ -486,6 +499,44 @@ public function userUploadsAFileTo($user, $source, $destination) } } + /** + * @When user :user uploads file with checksum :checksum and content :content to :destination + * @param $user + * @param $checksum + * @param $content + * @param $destination + */ + public function userUploadsAFileWithChecksumAndContentTo($user, $checksum, $content, $destination) + { + $file = \GuzzleHttp\Stream\Stream::factory($content); + try { + $this->response = $this->makeDavRequest( + $user, + "PUT", + $destination, + ['OC-Checksum' => $checksum], + $file + ); + } catch (\GuzzleHttp\Exception\BadResponseException $e) { + // 4xx and 5xx responses cause an exception + $this->response = $e->getResponse(); + } + } + + /** + * @Given file :file does not exist for user :user + * @param string $file + * @param $user + */ + public function fileDoesNotExist($file, $user) { + try { + $this->response = $this->makeDavRequest($user, 'DELETE', $file, []); + } catch (\GuzzleHttp\Exception\BadResponseException $e) { + // 4xx and 5xx responses cause an exception + $this->response = $e->getResponse(); + } + } + /** * @When User :user adds a file of :bytes bytes to :destination * @param string $user @@ -546,6 +597,27 @@ public function userCreatedAFolder($user, $destination) { } } + /** + * @Given user :user uploads new chunk file :num with :data to id :id with checksum :checksum + */ + public function userUploadsNewChunkFileOfWithToIdWithChecksum($user, $num, $data, $id, $checksum) + { + try { + $data = \GuzzleHttp\Stream\Stream::factory($data); + $destination = '/uploads/' . $user . '/' . $id . '/' . $num; + $this->makeDavRequest( + $user, + 'PUT', + $destination, + ['OC-Checksum' => $checksum], + $data, + "uploads" + ); + } catch (\GuzzleHttp\Exception\BadResponseException $ex) { + $this->response = $ex->getResponse(); + } + } + /** * @Given user :user uploads chunk file :num of :total with :data to :destination * @param string $user @@ -554,7 +626,7 @@ public function userCreatedAFolder($user, $destination) { * @param string $data * @param string $destination */ - public function userUploadsChunkFileOfWithToWithChecksum($user, $num, $total, $data, $destination) + public function userUploadsChunkedFile($user, $num, $total, $data, $destination) { $num -= 1; $data = \GuzzleHttp\Stream\Stream::factory($data); diff --git a/build/integration/features/checksums.feature b/build/integration/features/checksums.feature index d391e93afe8da..a2eeee8ece149 100644 --- a/build/integration/features/checksums.feature +++ b/build/integration/features/checksums.feature @@ -1,76 +1,187 @@ Feature: checksums Scenario: Uploading a file with checksum should work - Given user "user0" exists + Given using old dav path + And user "user0" exists When user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" Then The webdav response should have a status code "201" Scenario: Uploading a file with checksum should return the checksum in the propfind - Given user "user0" exists + Given using old dav path + And user "user0" exists And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" When user "user0" request the checksum of "/myChecksumFile.txt" via propfind - Then The webdav checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a" + Then The webdav checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f MD5:d70b40f177b14b470d1756a3c12b963a ADLER32:8ae90960" Scenario: Uploading a file with checksum should return the checksum in the download header - Given user "user0" exists + Given using old dav path + And user "user0" exists And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" When user "user0" downloads the file "/myChecksumFile.txt" - Then The header checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a" + Then The header checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f" Scenario: Moving a file with checksum should return the checksum in the propfind - Given user "user0" exists + Given using old dav path + And user "user0" exists And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" - When User "user0" moved file "/myChecksumFile.txt" to "/myMovedChecksumFile.txt" + When User "user0" moves file "/myChecksumFile.txt" to "/myMovedChecksumFile.txt" And user "user0" request the checksum of "/myMovedChecksumFile.txt" via propfind - Then The webdav checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a" + Then The webdav checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f MD5:d70b40f177b14b470d1756a3c12b963a ADLER32:8ae90960" Scenario: Moving file with checksum should return the checksum in the download header - Given user "user0" exists + Given using old dav path + And user "user0" exists + And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" + When User "user0" moves file "/myChecksumFile.txt" to "/myMovedChecksumFile.txt" + And user "user0" downloads the file "/myMovedChecksumFile.txt" + Then The header checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f" + + Scenario: Uploading a chunked file with checksum should return the checksum in the propfind + Given using old dav path + And user "user0" exists + And user "user0" uploads chunk file "1" of "3" with "AAAAA" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + And user "user0" uploads chunk file "2" of "3" with "BBBBB" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + And user "user0" uploads chunk file "3" of "3" with "CCCCC" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + When user "user0" request the checksum of "/myChecksumFile.txt" via propfind + Then The webdav checksum should match "SHA1:acfa6b1565f9710d4d497c6035d5c069bd35a8e8 MD5:45a72715acdd5019c5be30bdbb75233e ADLER32:1ecd03df" + + Scenario: Uploading a chunked file with checksum should return the checksum in the download header + Given using old dav path + And user "user0" exists + And user "user0" uploads chunk file "1" of "3" with "AAAAA" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + And user "user0" uploads chunk file "2" of "3" with "BBBBB" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + And user "user0" uploads chunk file "3" of "3" with "CCCCC" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + When user "user0" downloads the file "/myChecksumFile.txt" + Then The header checksum should match "SHA1:acfa6b1565f9710d4d497c6035d5c069bd35a8e8" + + @local_storage + Scenario: Downloading a file from local storage has correct checksum + Given using old dav path + And user "user0" exists + And file "prueba_cksum.txt" with text "Test file for checksums" is created in local storage + When user "user0" downloads the file "/local_storage/prueba_cksum.txt" + When user "user0" downloads the file "/local_storage/prueba_cksum.txt" + Then The header checksum should match "SHA1:a35b7605c8f586d735435535c337adc066c2ccb6" + + Scenario: Uploading a file with checksum should work using new dav path + Given using new dav path + And user "user0" exists + When user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" + Then The webdav response should have a status code "201" + + Scenario: Uploading a file with checksum should return the checksum in the propfind using new dav path + Given using new dav path + And user "user0" exists + And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" + When user "user0" request the checksum of "/myChecksumFile.txt" via propfind + Then The webdav checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f MD5:d70b40f177b14b470d1756a3c12b963a ADLER32:8ae90960" + + Scenario: Uploading a file with checksum should return the checksum in the download header using new dav path + Given using new dav path + And user "user0" exists + And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" + When user "user0" downloads the file "/myChecksumFile.txt" + Then The header checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f" + + Scenario: Moving a file with checksum should return the checksum in the propfind using new dav path + Given using new dav path + And user "user0" exists + And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" + When User "user0" moved file "/myChecksumFile.txt" to "/myMovedChecksumFile.txt" + And user "user0" request the checksum of "/myMovedChecksumFile.txt" via propfind + Then The webdav checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f MD5:d70b40f177b14b470d1756a3c12b963a ADLER32:8ae90960" + + Scenario: Moving file with checksum should return the checksum in the download header using new dav path + Given using new dav path + And user "user0" exists And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" When User "user0" moved file "/myChecksumFile.txt" to "/myMovedChecksumFile.txt" And user "user0" downloads the file "/myMovedChecksumFile.txt" - Then The header checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a" + Then The header checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f" - Scenario: Copying a file with checksum should return the checksum in the propfind - Given user "user0" exists + Scenario: Copying a file with checksum should return the checksum in the propfind using new dav path + Given using new dav path + And user "user0" exists And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" - When User "user0" copied file "/myChecksumFile.txt" to "/myChecksumFileCopy.txt" + When User "user0" copies file "/myChecksumFile.txt" to "/myChecksumFileCopy.txt" And user "user0" request the checksum of "/myChecksumFileCopy.txt" via propfind - Then The webdav checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a" + Then The webdav checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f MD5:d70b40f177b14b470d1756a3c12b963a ADLER32:8ae90960" - Scenario: Copying file with checksum should return the checksum in the download header - Given user "user0" exists + Scenario: Copying file with checksum should return the checksum in the download header using new dav path + Given using new dav path + And user "user0" exists And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" - When User "user0" copied file "/myChecksumFile.txt" to "/myChecksumFileCopy.txt" + When User "user0" copies file "/myChecksumFile.txt" to "/myChecksumFileCopy.txt" And user "user0" downloads the file "/myChecksumFileCopy.txt" - Then The header checksum should match "MD5:d70b40f177b14b470d1756a3c12b963a" + Then The header checksum should match "SHA1:3ee962b839762adb0ad8ba6023a4690be478de6f" + + Scenario: Uploading a chunked file with checksum should return the checksum in the propfind using new dav path + Given using new dav path + And user "user0" exists + And user "user0" uploads chunk file "1" of "3" with "AAAAA" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + And user "user0" uploads chunk file "2" of "3" with "BBBBB" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + And user "user0" uploads chunk file "3" of "3" with "CCCCC" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + When user "user0" request the checksum of "/myChecksumFile.txt" via propfind + Then The webdav checksum should match "SHA1:acfa6b1565f9710d4d497c6035d5c069bd35a8e8 MD5:45a72715acdd5019c5be30bdbb75233e ADLER32:1ecd03df" - Scenario: Overwriting a file with checksum should remove the checksum and not return it in the propfind + Scenario: Uploading a chunked file with checksum should return the checksum in the download header using new dav path + Given using new dav path + And user "user0" exists + And user "user0" uploads chunk file "1" of "3" with "AAAAA" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + And user "user0" uploads chunk file "2" of "3" with "BBBBB" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + And user "user0" uploads chunk file "3" of "3" with "CCCCC" to "/myChecksumFile.txt" with checksum "MD5:45a72715acdd5019c5be30bdbb75233e" + When user "user0" downloads the file "/myChecksumFile.txt" + Then The header checksum should match "SHA1:acfa6b1565f9710d4d497c6035d5c069bd35a8e8" + + @local_storage + Scenario: Downloading a file from local storage has correct checksum using new dav path + Given using new dav path + And user "user0" exists + And file "prueba_cksum.txt" with text "Test file for checksums" is created in local storage + When user "user0" downloads the file "/local_storage/prueba_cksum.txt" + When user "user0" downloads the file "/local_storage/prueba_cksum.txt" + Then The header checksum should match "SHA1:a35b7605c8f586d735435535c337adc066c2ccb6" + + Scenario: Upload chunked file where checksum does not match + Given using new dav path + And user "user0" exists + And user "user0" creates a new chunking upload with id "chunking-42" + And user "user0" uploads new chunk file "2" with "BBBBB" to id "chunking-42" + And user "user0" uploads new chunk file "3" with "CCCCC" to id "chunking-42" with checksum "SHA1:f005ba11" + And user "user0" uploads new chunk file "1" with "AAAAA" to id "chunking-42" + And user "user0" moves new chunk file with id "chunking-42" to "/myChunkedFile.txt" + Then the HTTP status code should be "400" + + Scenario: Upload a file where checksum does not match + Given using old dav path Given user "user0" exists - And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" - When user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" - And user "user0" request the checksum of "/myChecksumFile.txt" via propfind - Then The webdav checksum should be empty + And file "/chksumtst.txt" does not exist for user "user0" + And user "user0" uploads file with checksum "SHA1:f005ba11" and content "Some Text" to "/chksumtst.txt" + Then the HTTP status code should be "400" - Scenario: Overwriting a file with checksum should remove the checksum and not return it in the download header + Scenario: Upload a file where checksum does match + Given using old dav path Given user "user0" exists - And user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" with checksum "MD5:d70b40f177b14b470d1756a3c12b963a" - When user "user0" uploads file "data/textfile.txt" to "/myChecksumFile.txt" - And user "user0" downloads the file "/myChecksumFile.txt" - Then The OC-Checksum header should not be there + And file "/chksumtst.txt" does not exist for user "user0" + And user "user0" uploads file with checksum "SHA1:ce5582148c6f0c1282335b87df5ed4be4b781399" and content "Some Text" to "/chksumtst.txt" + Then the HTTP status code should be "201" - Scenario: Uploading a chunked file with checksum should return the checksum in the propfind + Scenario: Uploaded file should have the same checksum when downloaded + Given using old dav path Given user "user0" exists - And user "user0" uploads chunk file "1" of "3" with "AAAAA" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88" - And user "user0" uploads chunk file "2" of "3" with "BBBBB" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88" - And user "user0" uploads chunk file "3" of "3" with "CCCCC" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88" - When user "user0" request the checksum of "/myChecksumFile.txt" via propfind - Then The webdav checksum should match "MD5:e892fdd61a74bc89cd05673cc2e22f88" + And file "/chksumtst.txt" does not exist for user "user0" + And user "user0" uploads file with checksum "SHA1:ce5582148c6f0c1282335b87df5ed4be4b781399" and content "Some Text" to "/chksumtst.txt" + When Downloading file "/chksumtst.txt" as "user0" + Then The following headers should be set + | OC-Checksum | SHA1:ce5582148c6f0c1282335b87df5ed4be4b781399 | - Scenario: Uploading a chunked file with checksum should return the checksum in the download header + @local_storage + Scenario: Uploaded file to external storage should have the same checksum when downloaded + Given using old dav path Given user "user0" exists - And user "user0" uploads chunk file "1" of "3" with "AAAAA" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88" - And user "user0" uploads chunk file "2" of "3" with "BBBBB" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88" - And user "user0" uploads chunk file "3" of "3" with "CCCCC" to "/myChecksumFile.txt" with checksum "MD5:e892fdd61a74bc89cd05673cc2e22f88" - When user "user0" downloads the file "/myChecksumFile.txt" - Then The header checksum should match "MD5:e892fdd61a74bc89cd05673cc2e22f88" + And file "/local_storage/chksumtst.txt" does not exist for user "user0" + And user "user0" uploads file with checksum "SHA1:ce5582148c6f0c1282335b87df5ed4be4b781399" and content "Some Text" to "/local_storage/chksumtst.txt" + When Downloading file "/local_storage/chksumtst.txt" as "user0" + Then The following headers should be set + | OC-Checksum | SHA1:ce5582148c6f0c1282335b87df5ed4be4b781399 | + diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 9dea4d10fb225..e9d16e8f59f22 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -601,12 +601,14 @@ 'OC\\Files\\Storage\\StorageFactory' => $baseDir . '/lib/private/Files/Storage/StorageFactory.php', 'OC\\Files\\Storage\\Temporary' => $baseDir . '/lib/private/Files/Storage/Temporary.php', 'OC\\Files\\Storage\\Wrapper\\Availability' => $baseDir . '/lib/private/Files/Storage/Wrapper/Availability.php', + 'OC\\Files\\Storage\\Wrapper\\Checksum' => $baseDir . '/lib/private/Files/Storage/Wrapper/Checksum.php', 'OC\\Files\\Storage\\Wrapper\\Encoding' => $baseDir . '/lib/private/Files/Storage/Wrapper/Encoding.php', 'OC\\Files\\Storage\\Wrapper\\Encryption' => $baseDir . '/lib/private/Files/Storage/Wrapper/Encryption.php', 'OC\\Files\\Storage\\Wrapper\\Jail' => $baseDir . '/lib/private/Files/Storage/Wrapper/Jail.php', 'OC\\Files\\Storage\\Wrapper\\PermissionsMask' => $baseDir . '/lib/private/Files/Storage/Wrapper/PermissionsMask.php', 'OC\\Files\\Storage\\Wrapper\\Quota' => $baseDir . '/lib/private/Files/Storage/Wrapper/Quota.php', 'OC\\Files\\Storage\\Wrapper\\Wrapper' => $baseDir . '/lib/private/Files/Storage/Wrapper/Wrapper.php', + 'OC\\Files\\Stream\\Checksum' => $baseDir . '/lib/private/Files/Stream/Checksum.php', 'OC\\Files\\Stream\\Encryption' => $baseDir . '/lib/private/Files/Stream/Encryption.php', 'OC\\Files\\Stream\\Quota' => $baseDir . '/lib/private/Files/Stream/Quota.php', 'OC\\Files\\Type\\Detection' => $baseDir . '/lib/private/Files/Type/Detection.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 11d949de34a1e..d11ea437283db 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -631,12 +631,14 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\Storage\\StorageFactory' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/StorageFactory.php', 'OC\\Files\\Storage\\Temporary' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Temporary.php', 'OC\\Files\\Storage\\Wrapper\\Availability' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Availability.php', + 'OC\\Files\\Storage\\Wrapper\\Checksum' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Checksum.php', 'OC\\Files\\Storage\\Wrapper\\Encoding' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Encoding.php', 'OC\\Files\\Storage\\Wrapper\\Encryption' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Encryption.php', 'OC\\Files\\Storage\\Wrapper\\Jail' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Jail.php', 'OC\\Files\\Storage\\Wrapper\\PermissionsMask' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/PermissionsMask.php', 'OC\\Files\\Storage\\Wrapper\\Quota' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Quota.php', 'OC\\Files\\Storage\\Wrapper\\Wrapper' => __DIR__ . '/../../..' . '/lib/private/Files/Storage/Wrapper/Wrapper.php', + 'OC\\Files\\Stream\\Checksum' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Checksum.php', 'OC\\Files\\Stream\\Encryption' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Encryption.php', 'OC\\Files\\Stream\\Quota' => __DIR__ . '/../../..' . '/lib/private/Files/Stream/Quota.php', 'OC\\Files\\Type\\Detection' => __DIR__ . '/../../..' . '/lib/private/Files/Type/Detection.php', diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 1f3f2433e45cc..e7cb8d494e369 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -439,6 +439,9 @@ public function inCache($file) { */ public function remove($file) { $entry = $this->get($file); + if (!$entry || !$entry['fileid']) { + return; + } $sql = 'DELETE FROM `*PREFIX*filecache` WHERE `fileid` = ?'; $this->connection->executeQuery($sql, array($entry['fileid'])); if ($entry['mimetype'] === 'httpd/unix-directory') { diff --git a/lib/private/Files/Storage/Wrapper/Checksum.php b/lib/private/Files/Storage/Wrapper/Checksum.php new file mode 100644 index 0000000000000..ac186c3b43b88 --- /dev/null +++ b/lib/private/Files/Storage/Wrapper/Checksum.php @@ -0,0 +1,143 @@ + + * + * @copyright Copyright (c) 2017, ownCloud GmbH. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\Files\Storage\Wrapper; + +use Icewind\Streams\CallbackWrapper; +use OC\Cache\CappedMemoryCache; +use OC\Files\Stream\Checksum as ChecksumStream; +use OC\OCS\Exception; +use OCP\ILogger; +use OCP\Files\IHomeStorage; + +/** + * Class Checksum + * + * Computes checksums (default: SHA1, MD5, ADLER32) on all files under the /files path. + * The resulting checksum can be retrieved by call getMetadata($path) + * + * If a file is read and has no checksum oc_filecache gets updated accordingly. + * + * + * @package OC\Files\Storage\Wrapper + */ +class Checksum extends Wrapper { + + + const NOT_REQUIRED = 0; + /** Calculate checksum on write (to be stored in oc_filecache) */ + const PATH_NEW_OR_UPDATED = 1; + /** File needs to be checksummed on first read because it is already in cache but has no checksum */ + const PATH_IN_CACHE_WITHOUT_CHECKSUM = 2; + + /** + * @param string $path + * @param string $mode + * @return false|resource + */ + public function fopen($path, $mode) { + $stream = $this->getWrapperStorage()->fopen($path, $mode); + if (!is_resource($stream)) { + // don't wrap on error + return $stream; + } + + $requirement = $this->getChecksumRequirement($path, $mode); + + if ($requirement === self::PATH_NEW_OR_UPDATED || + $requirement === self::PATH_IN_CACHE_WITHOUT_CHECKSUM + ) { + return $this->wrapChecksumStream($stream, $path); + } + + return $stream; + } + + private function wrapChecksumStream($stream, $path) { + return \OC\Files\Stream\Checksum::wrap($stream, function (array $hashes) use ($path) { + $cache = $this->getCache(); + $cache->put($path, [ + 'checksum' => self::getChecksumsInDbFormat($hashes) + ]); + }); + } + + /** + * @param $mode + * @param $path + * @return int + */ + private function getChecksumRequirement($path, $mode) { + $isNormalFile = true; + if ($this->instanceOfStorage(IHomeStorage::class)) { + // home storage stores files in "files" + $isNormalFile = substr($path, 0, 6) === 'files/'; + } + $fileIsWritten = $mode !== 'r' && $mode !== 'rb'; + + if ($isNormalFile && $fileIsWritten) { + return self::PATH_NEW_OR_UPDATED; + } + + // file could be in cache but without checksum for example + // if mounted from ext. storage + $cache = $this->getCache($path); + + $cacheEntry = $cache->get($path); + + // Cache entry is sometimes an array (partial) when encryption is enabled without id so + // we ignore it. + if ($cacheEntry && empty($cacheEntry['checksum']) && is_object($cacheEntry)) { + return self::PATH_IN_CACHE_WITHOUT_CHECKSUM; + } + + return self::NOT_REQUIRED; + } + + /** + * @param array $hashes + * @return string + */ + private static function getChecksumsInDbFormat(array $hashes) { + $checksumString = ''; + foreach ($hashes as $algo => $checksum) { + $checksumString .= sprintf('%s:%s ', strtoupper($algo), $checksum); + } + + return rtrim($checksumString); + } + + /** + * @param string $path + * @param string $data + * @return bool + */ + public function file_put_contents($path, $data) { + $fh = $this->fopen($path, 'w'); + if (!$fh) { + return false; + } + fwrite($fh, $data); + fclose($fh); + + return true; + } +} diff --git a/lib/private/Files/Stream/Checksum.php b/lib/private/Files/Stream/Checksum.php new file mode 100644 index 0000000000000..975119bb28ca0 --- /dev/null +++ b/lib/private/Files/Stream/Checksum.php @@ -0,0 +1,153 @@ + + * + * @copyright Copyright (c) 2017, ownCloud GmbH. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\Files\Stream; + + +use Icewind\Streams\Wrapper; +use OC\Cache\CappedMemoryCache; + +/** + * Computes the checksum of the wrapped stream. The checksum can be retrieved with + * getChecksum after the stream is closed. + * + * + * @package OC\Files\Stream + */ +class Checksum extends Wrapper { + + /** + * To stepwise compute a hash on a continuous stream + * of data a "context" is required which stores the intermediate + * hash result while the stream has not finished. + * + * After the stream ends the hashing contexts needs to be finalized + * to compute the final checksum. + * + * @var resource[] + */ + private $hashingContexts; + + /** @var callable */ + private $callback; + + public function __construct(array $algos = ['sha1', 'md5', 'adler32']) { + + foreach ($algos as $algo) { + $this->hashingContexts[$algo] = hash_init($algo); + } + } + + + /** + * @param resource $source + * @param callable $callback + * @return resource + */ + public static function wrap($source, callable $callback) { + $context = stream_context_create([ + 'occhecksum' => [ + 'source' => $source, + 'callback' => $callback + ] + ]); + + return Wrapper::wrapSource( + $source, $context, 'occhecksum', self::class + ); + } + + + /** + * @param string $path + * @param array $options + * @return bool + */ + public function dir_opendir($path, $options) { + return false; + } + + /** + * @param string $path + * @param string $mode + * @param int $options + * @param string $opened_path + * @return bool + */ + public function stream_open($path, $mode, $options, &$opened_path) { + $context = parent::loadContext('occhecksum'); + $this->setSourceStream($context['source']); + $this->callback = $context['callback']; + + return true; + } + + /** + * @param int $count + * @return string + */ + public function stream_read($count) { + $data = parent::stream_read($count); + $this->updateHashingContexts($data); + + return $data; + } + + /** + * @param string $data + * @return int + */ + public function stream_write($data) { + $this->updateHashingContexts($data); + + return parent::stream_write($data); + } + + private function updateHashingContexts($data) { + foreach ($this->hashingContexts as $ctx) { + hash_update($ctx, $data); + } + } + + /** + * @return bool + */ + public function stream_close() { + $callback = $this->callback; + $callback($this->finalizeHashingContexts()); + + return parent::stream_close(); + } + + /** + * @return array + */ + private function finalizeHashingContexts() { + $hashes = []; + + foreach ($this->hashingContexts as $algo => $ctx) { + $hashes[$algo] = hash_final($ctx); + } + + return $hashes; + } +} diff --git a/lib/private/legacy/util.php b/lib/private/legacy/util.php index 9516a67af48c0..9fb974b47f086 100644 --- a/lib/private/legacy/util.php +++ b/lib/private/legacy/util.php @@ -180,6 +180,17 @@ public static function setupFS($user = '') { return $storage; }); + // install storage checksum wrapper + \OC\Files\Filesystem::addStorageWrapper('oc_checksum', function ($mountPoint, \OCP\Files\Storage\IStorage $storage) { + if (!$storage->instanceOfStorage('\OCA\Files_Sharing\SharedStorage')) { + return new \OC\Files\Storage\Wrapper\Checksum(['storage' => $storage]); + } + + return $storage; + + }, 1); + + \OC\Files\Filesystem::addStorageWrapper('oc_encoding', function ($mountPoint, \OCP\Files\Storage $storage, \OCP\Files\Mount\IMountPoint $mount) { if ($mount->getOption('encoding_compatibility', false) && !$storage->instanceOfStorage('\OCA\Files_Sharing\SharedStorage') && !$storage->isLocal()) { return new \OC\Files\Storage\Wrapper\Encoding(['storage' => $storage]); diff --git a/tests/lib/Files/ViewTest.php b/tests/lib/Files/ViewTest.php index f1e1ee7d417a0..d85969e7d194f 100644 --- a/tests/lib/Files/ViewTest.php +++ b/tests/lib/Files/ViewTest.php @@ -1755,6 +1755,8 @@ public function basicOperationProviderForLocks() { ILockingProvider::LOCK_SHARED, ILockingProvider::LOCK_SHARED, null, + // shared lock stays until fclose + ILockingProvider::LOCK_SHARED, ], [ 'opendir', @@ -1829,9 +1831,12 @@ public function testLockBasicOperation( $storage->expects($this->once()) ->method($operation) ->will($this->returnCallback( - function () use ($view, $lockedPath, &$lockTypeDuring) { + function () use ($view, $lockedPath, &$lockTypeDuring, $operation) { $lockTypeDuring = $this->getFileLockType($view, $lockedPath); + if ($operation === 'fopen') { + return fopen('data://text/plain,test', 'r'); + } return true; } )); @@ -1841,7 +1846,7 @@ function () use ($view, $lockedPath, &$lockTypeDuring) { $this->connectMockHooks($hookType, $view, $lockedPath, $lockTypePre, $lockTypePost); // do operation - call_user_func_array(array($view, $operation), $operationArgs); + $result = call_user_func_array([$view, $operation], $operationArgs); if ($hookType !== null) { $this->assertEquals($expectedLockBefore, $lockTypePre, 'File locked properly during pre-hook'); @@ -1852,6 +1857,13 @@ function () use ($view, $lockedPath, &$lockTypeDuring) { } $this->assertEquals($expectedStrayLock, $this->getFileLockType($view, $lockedPath)); + + if (is_resource($result)) { + fclose($result); + + // lock is cleared after fclose + $this->assertNull($this->getFileLockType($view, $lockedPath)); + } } /** @@ -2624,4 +2636,23 @@ public function testCreateParentDirectoriesWithExistingFile() { ->willReturn(true); $this->assertFalse(self::invokePrivate($view, 'createParentDirectories', ['/file.txt/folder/structure'])); } + + public function testFopenFail() { + // since stream wrappers influence the streams, + // this test makes sure that all stream wrappers properly return a failure + // to the caller instead of wrapping the boolean + /** @var Temporary | \PHPUnit_Framework_MockObject_MockObject $storage */ + $storage = $this->getMockBuilder(Temporary::class) + ->setMethods(['fopen']) + ->getMock(); + + $storage->expects($this->once()) + ->method('fopen') + ->willReturn(false); + + Filesystem::mount($storage, [], '/'); + $view = new View('/files'); + $result = $view->fopen('unexist.txt', 'r'); + $this->assertFalse($result); + } }