diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index 63a06b2cc95f..bf3535d5ee06 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -27,6 +27,7 @@ OCA\Files\Command\Scan OCA\Files\Command\DeleteOrphanedFiles OCA\Files\Command\TransferOwnership + OCA\Files\Command\VerifyChecksums diff --git a/apps/files/lib/Command/VerifyChecksums.php b/apps/files/lib/Command/VerifyChecksums.php new file mode 100644 index 000000000000..227a1aeba415 --- /dev/null +++ b/apps/files/lib/Command/VerifyChecksums.php @@ -0,0 +1,205 @@ + + * + * @copyright Copyright (c) 2018, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCA\Files\Command; + + +use OC\Files\FileInfo; +use OC\Files\Storage\Wrapper\Checksum; +use OCP\Files\IRootFolder; +use OCP\Files\Node; +use OCP\Files\NotFoundException; +use OCP\Files\Storage\IStorage; +use OCP\IUser; +use OCP\IUserManager; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + + +/** + * Recomputes checksums for all files and compares them to filecache + * entries. Provides repair option on mismatch. + * + * @package OCA\Files\Command + */ +class VerifyChecksums extends Command { + + + const EXIT_NO_ERRORS = 0; + const EXIT_CHECKSUM_ERRORS = 1; + const EXIT_INVALID_ARGS = 2; + + /** + * @var IRootFolder + */ + private $rootFolder; + + /** + * @var IUserManager + */ + private $userManager; + + private $exitStatus = self::EXIT_NO_ERRORS; + + /** + * VerifyChecksums constructor. + * + * @param IRootFolder $rootFolder + * @param IUserManager $userManager + */ + public function __construct(IRootFolder $rootFolder, IUserManager $userManager) { + parent::__construct(null); + $this->rootFolder = $rootFolder; + $this->userManager = $userManager; + } + + protected function configure() { + $this + ->setName('files:checksums:verify') + ->setDescription('Get all checksums in filecache and compares them by recalculating the checksum of the file.') + ->addOption('repair', 'r', InputOption::VALUE_NONE, 'Repair filecache-entry with mismatched checksums.') + ->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Specific user to check') + ->addOption('path', 'p', InputOption::VALUE_REQUIRED, 'Path to check relative to data e.g /john/files/', ''); + } + + public function execute(InputInterface $input, OutputInterface $output) { + $pathOption = $input->getOption('path'); + $userName = $input->getOption('user'); + + if (!$pathOption && !$userName) { + $output->writeln('This operation might take very long.'); + } + + if ($pathOption && $userName) { + $output->writeln('Please use either path or user exclusively'); + $this->exitStatus = self::EXIT_INVALID_ARGS; + + } + + $walkFunction = function (Node $node) use ($input, $output) { + $path = $node->getInternalPath(); + $currentChecksums = $node->getChecksum(); + + // Files without calculated checksum can't cause checksum errors + if (empty($currentChecksums)) { + $output->writeln("Skipping $path => No Checksum", OutputInterface::VERBOSITY_VERBOSE); + return; + } + + $output->writeln("Checking $path => $currentChecksums", OutputInterface::VERBOSITY_VERBOSE); + $actualChecksums = self::calculateActualChecksums($path, $node->getStorage()); + + if ($actualChecksums !== $currentChecksums) { + $output->writeln( + "Mismatch for $path:\n Filecache:\t$currentChecksums\n Actual:\t$actualChecksums" + ); + + $this->exitStatus = self::EXIT_CHECKSUM_ERRORS; + + if ($input->getOption('repair')) { + $output->writeln("Repairing $path"); + $this->updateChecksumsForNode($node, $actualChecksums); + $this->exitStatus = self::EXIT_NO_ERRORS; + } + } + }; + + $scanUserFunction = function(IUser $user) use ($input, $output, $walkFunction) { + $userFolder = $this->rootFolder->getUserFolder($user->getUID())->getParent(); + $this->walkNodes($userFolder->getDirectoryListing(), $walkFunction); + }; + + if ($userName && $this->userManager->userExists($userName)) { + $scanUserFunction($this->userManager->get($userName)); + } else if ($userName && !$this->userManager->userExists($userName)) { + $output->writeln("User \"$userName\" does not exist"); + $this->exitStatus = self::EXIT_INVALID_ARGS; + } else if ($input->getOption('path')) { + + try { + $node = $this->rootFolder->get($input->getOption('path')); + } catch (NotFoundException $ex) { + $output->writeln("Path \"{$ex->getMessage()}\" not found."); + $this->exitStatus = self::EXIT_INVALID_ARGS; + return $this->exitStatus; + } + + $this->walkNodes([$node], $walkFunction); + + } else { + $this->userManager->callForAllUsers($scanUserFunction); + } + + + return $this->exitStatus; + } + + + /** + * Recursive walk nodes + * + * @param Node[] $nodes + * @param \Closure $callBack + */ + private function walkNodes(array $nodes, \Closure $callBack) { + foreach ($nodes as $node) { + if ($node->getType() === FileInfo::TYPE_FOLDER) { + $this->walkNodes($node->getDirectoryListing(), $callBack); + } else { + $callBack($node); + } + } + } + + /** + * @param Node $node + * @param $correctChecksum + * @throws NotFoundException + * @throws \OCP\Files\InvalidPathException + * @throws \OCP\Files\StorageNotAvailableException + */ + private function updateChecksumsForNode(Node $node, $correctChecksum) { + $storage = $node->getStorage(); + $cache = $storage->getCache(); + $cache->update( + $node->getId(), + ['checksum' => $correctChecksum] + ); + } + + /** + * @param $path + * @param IStorage $storage + * @return string + * @throws \OCP\Files\StorageNotAvailableException + */ + private static function calculateActualChecksums($path, IStorage $storage) { + return sprintf( + Checksum::CHECKSUMS_DB_FORMAT, + $storage->hash('sha1', $path), + $storage->hash('md5', $path), + $storage->hash('adler32', $path) + ); + } +} + diff --git a/apps/files/tests/Command/VerifyChecksumsTest.php b/apps/files/tests/Command/VerifyChecksumsTest.php new file mode 100644 index 000000000000..ac124b990715 --- /dev/null +++ b/apps/files/tests/Command/VerifyChecksumsTest.php @@ -0,0 +1,315 @@ +rootFolder = \OC::$server->getRootFolder(); + + $this->user1 = $this->createRandomUser(1); + $this->user2 = $this->createRandomUser(2); + + + $this->testFiles = [ + $this->createFileForUser($this->user1, 'dir/nested/somefile.txt', 'Hello World!'), + $this->createFileForUser($this->user1, 'dir/nested/subdir/hallo2.txt', 'ewfwfwefwef'), + $this->createFileForUser($this->user1, 'dir/nested/somefile2.txt', '1337'), + $this->createFileForUser($this->user1, 'otherdir/bar.txt', 'Bye!'), + $this->createFileForUser($this->user1, 'rootfile.doc', 'efwefwfwefwffw'), + $this->createFileForUser($this->user1, 'song.mp3', 'sdfsfdsdfsdfsfsfsffs'), + $this->createFileForUser($this->user2, 'sdvsdvs/secrets/bar/baz/moo.txt', 'JhonnyCache'), + $this->createFileForUser($this->user2, 'bling.tif', 'ewfwefwf242'), + $this->createFileForUser($this->user2, 'pong.tif', '12'), + $this->createFileForUser($this->user2, 'welcome.doc', 'efwefwfwefwffw'), + ]; + + $this->cmd = new CommandTester( + new VerifyChecksums( + $this->rootFolder, + \OC::$server->getUserManager() + ) + ); + } + + /** + * @param string $uid + * @param $path + * @param $content + * @return array + * @throws \OCP\Files\NotPermittedException + */ + private function createFileForUser($uid, $path, $content) { + + $userFolder = \OC::$server->getUserFolder($uid); + + $parts = explode('/', ltrim($path, '/')); + $fileName = array_pop($parts); + $dirPath = $parts; + + $currentDir = ''; + foreach ($dirPath as $subDir) { + if (!empty($subDir)) { + $currentDir = "$currentDir/$subDir"; + if (!$userFolder->nodeExists($currentDir)) { + $userFolder->newFolder("$currentDir"); + } + } + } + + $f = $userFolder->newFile("$currentDir/$fileName"); + $f->putContent($content); + + + + return [ + 'file' => $f, + 'expectedChecksums' => function() use ($content) { + return sprintf( + Checksum::CHECKSUMS_DB_FORMAT, + hash('sha1', $content), + hash('md5', $content), + hash('adler32', $content) + ); + }, + ]; + } + + + /** + * @param int $number + * @return bool|IUser + */ + private function createRandomUser($number) { + $userName = $this->getUniqueID("$number-verifycheksums"); + $user = $this->createUser($userName); + $this->loginAsUser($userName); + + return $user->getUID(); + } + + + private function breakChecksum(File &$f) { + $cache = $f->getStorage()->getCache(); + $cache->update($f->getId(), ['checksum' => self::BROKEN_CHECKSUM_STRING]); + $this->refreshFileInfo($f); + } + + private function refreshFileInfo(File &$f) { + $f = $this->rootFolder->get($f->getPath()); + } + + private function assertChecksumsAreCorrect(array $files) { + foreach ($files as $key => $file) { + /** @var File $f */ + $f = $files[$key]['file']; + $expectedChecksums = $files[$key]['expectedChecksums']; + $this->refreshFileInfo($files[$key]['file']); + $this->assertSame( + $expectedChecksums(), + $f->getChecksum() + ); + } + } + + + public function testNoBrokenChecksums() { + $this->cmd->execute([]); + $exitCode = $this->cmd->getStatusCode(); + + $this->assertEquals(VerifyChecksums::EXIT_NO_ERRORS, $exitCode, 'Wrong exit code'); + $this->assertChecksumsAreCorrect($this->testFiles); + } + + public function testBrokenChecksumsAreNotRepairedWithoutArguments() { + /** @var File $file1 */ + $file1 = $this->testFiles[0]['file']; + /** @var File $file2 */ + $file2 = $this->testFiles[6]['file']; + + $this->breakChecksum($file1); + $this->breakChecksum($file2); + + $this->cmd->execute([]); + $this->cmd->execute([]); + + + $exitCode = $this->cmd->getStatusCode(); + $this->assertEquals(VerifyChecksums::EXIT_CHECKSUM_ERRORS, $exitCode, 'Wrong exit code'); + + + $this->assertEquals(self::BROKEN_CHECKSUM_STRING, $file1->getChecksum()); + $this->assertEquals(self::BROKEN_CHECKSUM_STRING, $file2->getChecksum()); + } + + /** + * @depends testBrokenChecksumsAreNotRepairedWithoutArguments + */ + public function testFilesWithBrokenChecksumsAreDisplayed() { + /** @var File $file1 */ + $file1 = $this->testFiles[4]['file']; + /** @var File $file2 */ + $file2 = $this->testFiles[7]['file']; + + $this->breakChecksum($file1); + $this->breakChecksum($file2); + + $this->cmd->execute([]); + + $output = $this->cmd->getDisplay(); + + $this->assertContains("Mismatch for {$file1->getInternalPath()}", $output); + $this->assertContains("Mismatch for {$file2->getInternalPath()}", $output); + $this->assertContains(self::BROKEN_CHECKSUM_STRING, $output); + $this->assertContains($this->testFiles[4]['expectedChecksums'](), $output); + $this->assertContains($this->testFiles[7]['expectedChecksums'](), $output); + } + + /** + * @depends testNoBrokenChecksums + */ + public function testBrokenChecksumsResultInErrorExitCode() { + + /** @var File $file1 */ + $file1 = $this->testFiles[0]['file']; + + $this->breakChecksum($file1); + $this->cmd->execute([]); + + $this->assertEquals(self::BROKEN_CHECKSUM_STRING, $file1->getChecksum()); + + $exitCode = $this->cmd->getStatusCode(); + $this->assertEquals(VerifyChecksums::EXIT_CHECKSUM_ERRORS, $exitCode, 'Wrong exit code'); + + } + + /** + * @depends testBrokenChecksumsAreNotRepairedWithoutArguments + */ + public function testBrokenFilesAreRepairedWithRepairArgument() { + /** @var File $file1 */ + $file1 = $this->testFiles[1]['file']; + /** @var File $file2 */ + $file2 = $this->testFiles[7]['file']; + + $this->breakChecksum($file1); + $this->breakChecksum($file2); + + $this->cmd->execute(['-r' => null]); + + $this->assertChecksumsAreCorrect([ + $this->testFiles[1], + $this->testFiles[7] + ]); + + $exitCode = $this->cmd->getStatusCode(); + $this->assertEquals(VerifyChecksums::EXIT_NO_ERRORS, $exitCode, "Wrong exit code"); + } + + /** + * @depends testBrokenFilesAreRepairedWithRepairArgument + */ + public function testOnlyFilesInGivenPathArgumentAreRepaired() { + + /** @var File $file1 */ + $file1 = $this->testFiles[0]['file']; + /** @var File $file2 */ + $file2 = $this->testFiles[1]['file']; + /** @var File $file3 */ + $file3 = $this->testFiles[2]['file']; + /** @var File $file4 */ + $file4 = $this->testFiles[3]['file']; + + $this->breakChecksum($file1); + $this->breakChecksum($file2); + $this->breakChecksum($file3); + $this->breakChecksum($file4); + + $this->cmd->execute([ + '-r' => null, + '-p' => "{$this->user1}/files/dir/nested" + ]); + + $this->assertChecksumsAreCorrect([ + $this->testFiles[0], + $this->testFiles[1], + $this->testFiles[2], + ]); + + $this->assertEquals(self::BROKEN_CHECKSUM_STRING, $file4->getChecksum()); + } + + public function testOnlyFilesOfAGivenUserAreRepaired() { + /** @var File $file1 */ + $file1 = $this->testFiles[0]['file']; + /** @var File $file2 */ + $file2 = $this->testFiles[6]['file']; + + $this->breakChecksum($file1); + $this->breakChecksum($file2); + + $this->cmd->execute([ + '-r' => null, + '-u' => $this->user1 + ]); + + $this->assertChecksumsAreCorrect([ + $this->testFiles[0], + ]); + + $this->assertEquals(self::BROKEN_CHECKSUM_STRING, $file2->getChecksum()); + } + + public function testAllFilesCanBeRepaired() { + foreach ($this->testFiles as $testFile) { + $this->breakChecksum($testFile['file']); + } + + $this->cmd->execute(['-r' => null]); + + $this->assertChecksumsAreCorrect($this->testFiles); + } +} diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 249e01cb75c3..bf76d42492b6 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -205,6 +205,14 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = $fileId = -1; } if (!empty($newData)) { + // Only reset checksum on file change + foreach (array_intersect_key($newData, $data) as $key => $value) { + if (in_array($key, ['size', 'storage_mtime', 'mtime', 'etag']) && $data[$key] != $newData[$key]) { + $newData['checksum'] = ''; + } + } + + $data['fileid'] = $this->addToCache($file, $newData, $fileId); } if (isset($cacheData['size'])) { diff --git a/lib/private/Files/ObjectStore/NoopScanner.php b/lib/private/Files/ObjectStore/NoopScanner.php index dc65e924b9d5..a1182233a22b 100644 --- a/lib/private/Files/ObjectStore/NoopScanner.php +++ b/lib/private/Files/ObjectStore/NoopScanner.php @@ -30,6 +30,7 @@ class NoopScanner extends Scanner { public function __construct(Storage $storage) { + $this->storage = $storage; //we don't need the storage, so do nothing here } @@ -55,6 +56,15 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = * @return array with the meta data of the scanned file or folder */ public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { + // we only update the checksums - still returning no data + $meta = $this->storage->getMetaData($path); + if (isset($meta['checksum'])) { + $this->storage->getCache()->put( + $path, + ['checksum' => $meta['checksum']] + ); + } + return []; } diff --git a/lib/private/Files/Storage/Wrapper/Checksum.php b/lib/private/Files/Storage/Wrapper/Checksum.php index 7868768be1cf..bfdbe3b460f7 100644 --- a/lib/private/Files/Storage/Wrapper/Checksum.php +++ b/lib/private/Files/Storage/Wrapper/Checksum.php @@ -37,6 +37,8 @@ */ class Checksum extends Wrapper { + /** Format of checksum field in filecache */ + const CHECKSUMS_DB_FORMAT = 'SHA1:%s MD5:%s ADLER32:%s'; const NOT_REQUIRED = 0; /** Calculate checksum on write (to be stored in oc_filecache) */ @@ -133,16 +135,21 @@ public function onClose() { /** * @param $path - * Format like "SHA1:abc MD5:def ADLER32:ghi" - * @return string + * @return string Format like "SHA1:abc MD5:def ADLER32:ghi" */ private static function getChecksumsInDbFormat($path) { - $checksumString = ''; - foreach (ChecksumStream::getChecksums($path) as $algo => $checksum) { - $checksumString .= sprintf('%s:%s ', strtoupper($algo), $checksum); + $checksums = ChecksumStream::getChecksums($path); + + if (empty($checksums)) { + return ''; } - return rtrim($checksumString); + return sprintf( + self::CHECKSUMS_DB_FORMAT, + $checksums['sha1'], + $checksums['md5'], + $checksums['adler32'] + ); } /** diff --git a/tests/lib/Files/ViewTest.php b/tests/lib/Files/ViewTest.php index 42965cc23de0..17a5546994db 100644 --- a/tests/lib/Files/ViewTest.php +++ b/tests/lib/Files/ViewTest.php @@ -592,7 +592,6 @@ public function testRmdir($method) { $this->assertInstanceOf(GenericEvent::class, $calledCreateEvent[1]); $this->assertEquals('file.aftercreate', $calledCreateEvent[0]); - $this->assertFalse($rootView->file_exists('sub')); } @@ -2605,59 +2604,6 @@ public function testGetDirectoryContentMimeFilter($filter, $expected) { $this->assertEquals($expected, $files); } - private function calculateChecksums($data) { - // Get hashing algorithms and calculate checksum for given data - $checksumString = ''; - $checksums['sha1'] = hash('sha1', $data); - $checksums['md5'] = hash('md5', $data); - $checksums['adler32'] = hash('adler32', $data); - foreach ($checksums as $algo => $checksum) { - $checksumString .= sprintf('%s:%s ', strtoupper($algo), $checksum); - } - return rtrim($checksumString); - } - - public function testFilePutContentsClearsChecksum() { - $storage = new Temporary([]); - $scanner = $storage->getScanner(); - $storage->mkdir('files'); - $storage->file_put_contents('files/foo.txt', 'bar'); - Filesystem::mount($storage, [], '/files'); - $scanner->scan(''); - - $calledReadEvent = []; - \OC::$server->getEventDispatcher()->addListener('file.afterread', function ($event) use (&$calledReadEvent) { - $calledReadEvent[] = 'file.afterread'; - $calledReadEvent[] = $event; - }); - // Obtain view on the file and check content + checksum - // If we insert file using $storage->file_put_contents checksum is not being calculated (null) - $view = new View('/files'); - $this->assertEquals('bar', $view->file_get_contents('files/foo.txt')); - $this->assertEquals('file.afterread', $calledReadEvent[0]); - $this->assertArrayHasKey('path', $calledReadEvent[1]); - $this->assertInstanceOf(GenericEvent::class, $calledReadEvent[1]); - $data = $view->getFileInfo('files/foo.txt'); - $this->assertNull($data->getChecksum()); - - // Force some checksum in the file and verify - $view->putFileInfo('files/foo.txt', ['checksum' => '42']); - $data = $view->getFileInfo('files/foo.txt'); - $checksum = $data->getChecksum(); - $this->assertEquals('42', $checksum); - - // Now use Wrapper/Checksum to calculate checksum for us (normal PUT case) - $fh = tmpfile(); - fwrite($fh, 'fooo'); - rewind($fh); - $view->file_put_contents('files/foo.txt', $fh); - $this->assertEquals('fooo', $view->file_get_contents('files/foo.txt')); - $data = $view->getFileInfo('files/foo.txt'); - $checksum = $data->getChecksum(); - $expectedChecksum = $this->calculateChecksums('fooo'); - $this->assertEquals($expectedChecksum, $checksum); - } - public function testDeleteGhostFile() { $storage = new Temporary([]); $scanner = $storage->getScanner();