From 6b4d77bf4c5039ccb505ad4f09caac59de36d583 Mon Sep 17 00:00:00 2001 From: Louis Chemineau Date: Wed, 23 Mar 2022 15:19:08 +0100 Subject: [PATCH] Add occ command to repair mtime Signed-off-by: Louis Chemineau --- apps/files/appinfo/info.xml | 3 +- .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + apps/files/lib/Command/RepairMtime.php | 255 ++++++++++++++++++ apps/files/tests/Command/RepairMtimeTest.php | 129 +++++++++ .../lib/Lib/Storage/AmazonS3.php | 2 +- lib/private/Files/ObjectStore/Azure.php | 6 + .../Files/ObjectStore/S3ObjectTrait.php | 8 + .../Files/ObjectStore/StorageObjectStore.php | 4 + lib/private/Files/ObjectStore/Swift.php | 6 + lib/public/Files/ObjectStore/IObjectStore.php | 7 + .../ObjectStore/FailDeleteObjectStore.php | 4 + .../ObjectStore/FailWriteObjectStore.php | 4 + 13 files changed, 428 insertions(+), 2 deletions(-) create mode 100644 apps/files/lib/Command/RepairMtime.php create mode 100644 apps/files/tests/Command/RepairMtimeTest.php diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index 0bfeed2b3a0f2..38c701d521099 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -35,6 +35,7 @@ OCA\Files\Command\TransferOwnership OCA\Files\Command\ScanAppData OCA\Files\Command\RepairTree + OCA\Files\Command\RepairMtime @@ -67,4 +68,4 @@ OCA\Files\Settings\PersonalSettings - + \ No newline at end of file diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php index bc2e496294b4e..ce725eadf1001 100644 --- a/apps/files/composer/composer/autoload_classmap.php +++ b/apps/files/composer/composer/autoload_classmap.php @@ -27,6 +27,7 @@ 'OCA\\Files\\Collaboration\\Resources\\Listener' => $baseDir . '/../lib/Collaboration/Resources/Listener.php', 'OCA\\Files\\Collaboration\\Resources\\ResourceProvider' => $baseDir . '/../lib/Collaboration/Resources/ResourceProvider.php', 'OCA\\Files\\Command\\DeleteOrphanedFiles' => $baseDir . '/../lib/Command/DeleteOrphanedFiles.php', + 'OCA\\Files\\Command\\RepairMtime' => $baseDir . '/../lib/Command/RepairMtime.php', 'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php', 'OCA\\Files\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php', 'OCA\\Files\\Command\\ScanAppData' => $baseDir . '/../lib/Command/ScanAppData.php', diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php index ba39b2c57073a..df07e90428635 100644 --- a/apps/files/composer/composer/autoload_static.php +++ b/apps/files/composer/composer/autoload_static.php @@ -42,6 +42,7 @@ class ComposerStaticInitFiles 'OCA\\Files\\Collaboration\\Resources\\Listener' => __DIR__ . '/..' . '/../lib/Collaboration/Resources/Listener.php', 'OCA\\Files\\Collaboration\\Resources\\ResourceProvider' => __DIR__ . '/..' . '/../lib/Collaboration/Resources/ResourceProvider.php', 'OCA\\Files\\Command\\DeleteOrphanedFiles' => __DIR__ . '/..' . '/../lib/Command/DeleteOrphanedFiles.php', + 'OCA\\Files\\Command\\RepairMtime' => __DIR__ . '/..' . '/../lib/Command/RepairMtime.php', 'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php', 'OCA\\Files\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php', 'OCA\\Files\\Command\\ScanAppData' => __DIR__ . '/..' . '/../lib/Command/ScanAppData.php', diff --git a/apps/files/lib/Command/RepairMtime.php b/apps/files/lib/Command/RepairMtime.php new file mode 100644 index 0000000000000..fe8ffbb799e1b --- /dev/null +++ b/apps/files/lib/Command/RepairMtime.php @@ -0,0 +1,255 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ +namespace OCA\Files\Command; + +use Symfony\Component\Console\Helper\Table; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; +use OC\ForbiddenException; +use OC\Core\Command\Base; +use OC\Core\Command\InterruptedException; +use OC\Files\Search\SearchComparison; +use OC\Files\Search\SearchOrder; +use OC\Files\Search\SearchQuery; +use OCP\Files\Search\ISearchComparison; +use OCP\Files\Search\ISearchOrder; +use OCP\Files\NotFoundException; +use OCP\Files\IRootFolder; +use OCP\IUserManager; +use OCP\IDBConnection; + +class RepairMtime extends Base { + private IUserManager $userManager; + private IRootFolder $rootFolder; + protected IDBConnection $connection; + + protected float $execTime = 0; + protected int $filesCounter = 0; + + public function __construct(IDBConnection $connection, IUserManager $userManager, IRootFolder $rootFolder) { + $this->connection = $connection; + $this->userManager = $userManager; + $this->rootFolder = $rootFolder; + parent::__construct(); + } + + protected function configure(): void { + parent::configure(); + + $this + ->setName('files:repair-mtime') + ->setDescription('Repair files\' mtime') + ->addArgument( + 'user_id', + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + 'will repair mtime for all files of the given user(s)' + ) + ->addOption( + 'all', + null, + InputOption::VALUE_NONE, + 'will repair all files of all known users' + ) + ->addOption( + 'dry-run', + null, + InputOption::VALUE_NONE, + 'will list files instead of repairing them' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + if ($input->getOption('all')) { + $users = $this->userManager->search(''); + } else { + $users = $input->getArgument('user_id'); + } + + # check quantity of users to be process and show it on the command line + $users_total = count($users); + if ($users_total === 0) { + $output->writeln('Please specify the user id or --all for all users'); + return 1; + } + + $this->initTools(); + + $user_count = 0; + foreach ($users as $user) { + if (is_object($user)) { + $user = $user->getUID(); + } + ++$user_count; + if ($this->userManager->userExists($user)) { + $this->repairMtimeForUser( + $user, + $input->getOption('dry-run'), + $output, + ); + } else { + $output->writeln("Unknown user $user_count $user"); + } + + try { + $this->abortIfInterrupted(); + } catch (InterruptedException $e) { + break; + } + } + + $this->presentStats($output, $input->getOption('dry-run')); + return 0; + } + + public function repairMtimeForUser(string $userId, bool $dryRun, OutputInterface $output): void { + $userFolder = $this->rootFolder->getUserFolder($userId); + $user = $this->userManager->get($userId); + + $offset = 0; + + do { + $invalidFiles = $userFolder + ->search( + new SearchQuery( + new SearchComparison(ISearchComparison::COMPARE_LESS_THAN_EQUAL, 'mtime', 86400), + 0, // 0 = no limits. + $offset, + [new SearchOrder(ISearchOrder::DIRECTION_DESCENDING, 'mtime')], + $user + ) + ); + + $offset += count($invalidFiles); + + $this->connection->beginTransaction(); + + foreach ($invalidFiles as $file) { + $this->filesCounter++; + + try { + $filePath = $file->getPath(); + $fileId = $file->getId(); + $fileStorage = $file->getStorage(); + + // Default new mtime to the current time. + $mtime = time(); + + if ($fileStorage->instanceOfStorage(\OC\Files\ObjectStore\ObjectStoreStorage::class)) { + // Get LastModified property for S3 as primary storage. + /** @var \OC\Files\ObjectStore\ObjectStoreStorage $fileStorage */ + $headResult = $fileStorage->getObjectStore()->headObject("urn:oid:$fileId"); + if ($headResult !== false) { + $date = \DateTime::createFromFormat(\DateTimeInterface::ISO8601, $headResult['LastModified']); + $mtime = $date->getTimestamp(); + } + } elseif ($file->getStorage()->instanceOfStorage(\OCA\Files_External\Lib\Storage\AmazonS3::class)) { + // Get LastModified property for S3 as external storage. + /** @var \OCA\Files_External\Lib\Storage\AmazonS3 $fileStorage */ + $headResult = $fileStorage->headObject("urn:oid:$fileId"); + if ($headResult !== false) { + $date = \DateTime::createFromFormat(\DateTimeInterface::ISO8601, $headResult['LastModified']); + $mtime = $date->getTimestamp(); + } + } + + $humanMtime = date(DATE_RFC2822, $mtime); + if ($dryRun) { + $output->writeln("- Found '$filePath', would set the mtime to $mtime ($humanMtime).", OutputInterface::VERBOSITY_VERBOSE); + } else { + $file->touch($mtime); + $output->writeln("- Fixed $filePath with $mtime ($humanMtime)", OutputInterface::VERBOSITY_VERBOSE); + } + } catch (ForbiddenException $e) { + $output->writeln("Home storage for user $userId not writable"); + $output->writeln('Make sure you\'re running the command only as the user the web server runs as'); + } catch (InterruptedException $e) { + # exit the function if ctrl-c has been pressed + $output->writeln('Interrupted by user'); + } catch (NotFoundException $e) { + $output->writeln('Path not found: ' . $e->getMessage() . ''); + } catch (\Exception $e) { + $output->writeln('Exception: ' . $e->getMessage() . ''); + $output->writeln('' . $e->getTraceAsString() . ''); + } + } + + $this->connection->commit(); + } while (count($invalidFiles) > 0); + } + + /** + * Initialises some useful tools for the Command + */ + protected function initTools(): void { + // Start the timer + $this->execTime = -microtime(true); + // Convert PHP errors to exceptions + set_error_handler([$this, 'exceptionErrorHandler'], E_ALL); + } + + /** + * Processes PHP errors as exceptions in order to be able to keep track of problems + * + * @see https://www.php.net/manual/en/function.set-error-handler.php + * + * @param int $severity the level of the error raised + * @param string $message + * @param string $file the filename that the error was raised in + * @param int $line the line number the error was raised + * + * @throws \ErrorException + */ + public function exceptionErrorHandler(int $severity, string $message, string $file, int $line): void { + if (!(error_reporting() & $severity)) { + // This error code is not included in error_reporting + return; + } + throw new \ErrorException($message, 0, $severity, $file, $line); + } + + protected function presentStats(OutputInterface $output, bool $dryRun): void { + // Stop the timer + $this->execTime += microtime(true); + + $columnName = 'Fixed files'; + if ($dryRun) { + $columnName = 'Found files'; + } + + $table = new Table($output); + $table + ->setHeaders([$columnName, 'Elapsed time']) + ->setRows([[$this->filesCounter, $this->formatExecTime()]]) + ->render(); + } + + /** + * Formats microtime into a human readable format + */ + protected function formatExecTime(): string { + $secs = round($this->execTime); + # convert seconds into HH:MM:SS form + return sprintf('%02d:%02d:%02d', ($secs / 3600), ($secs / 60 % 60), $secs % 60); + } +} diff --git a/apps/files/tests/Command/RepairMtimeTest.php b/apps/files/tests/Command/RepairMtimeTest.php new file mode 100644 index 0000000000000..1db50d1d6393d --- /dev/null +++ b/apps/files/tests/Command/RepairMtimeTest.php @@ -0,0 +1,129 @@ + + * Copyright (c) 2014-2015 Olivier Paroz owncloud@oparoz.com + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OCA\Files\Tests\Command; + +use OCP\IDBConnection; +use OCP\IUserManager; +use OCP\Files\IRootFolder; +use \OCA\Files\Command\RepairMtime; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Formatter\OutputFormatterInterface; + +/** + * Tests for the repairing invalid mtime. + * + * @group DB + * + * @see \OCA\Files\Command\RepairMtime + */ +class RepairMtimeTest extends \Test\TestCase { + private IDBConnection $connection; + private IUserManager $userManager; + private IRootFolder $rootFolder; + + private RepairMtime $repairMtime; + + /** + * @var \PHPUnit\Framework\MockObject\MockObject|InputInterface + */ + private InputInterface $inputMock; + /** + * @var \PHPUnit\Framework\MockObject\MockObject|InputInterface + */ + private InputInterface $inputDryRunMock; + + protected function setUp(): void { + parent::setUp(); + + \OC::$server->getUserManager()->createUser('user1-mtime-repair', 'password'); + + $this->connection = \OC::$server->get(IDBConnection::class); + $this->userManager = \OC::$server->get(IUserManager::class); + $this->rootFolder = \OC::$server->get(IRootFolder::class); + + $this->repairMtime = new \OCA\Files\Command\RepairMtime($this->connection, $this->userManager, $this->rootFolder); + + $this->inputMock = $this->createMock(InputInterface::class); + $this->inputMock + ->expects($this->any()) + ->method('getArgument') + ->willReturnMap([['user_id', ['user1-mtime-repair']]]); + $this->inputMock + ->expects($this->any()) + ->method('getOption') + ->willReturnMap([['path', ''], ['dry-run', false]]); + + $this->inputDryRunMock = $this->createMock(InputInterface::class); + $this->inputDryRunMock + ->expects($this->any()) + ->method('getArgument') + ->willReturnMap([['user_id', ['user1-mtime-repair']]]); + $this->inputDryRunMock + ->expects($this->any()) + ->method('getOption') + ->willReturnMap([['path', ''], ['dry-run', true]]); + } + + public function tearDown(): void { + \OC::$server->getUserManager()->get('user1-mtime-repair')->delete(); + } + + public function testRepairMtimeLocalFile() { + $userFolder = $this->rootFolder->getUserFolder('user1-mtime-repair'); + + for ($i = 0; $i < 10; $i++) { + $userFolder + ->newFile("file_nb_$i.txt", "file_content_$i") + ->touch(0); + } + + $found = 0; + $fixed = 0; + + /** + * @var \PHPUnit\Framework\MockObject\MockObject|OutputInterface + */ + $outputMock = $this->createMock(OutputInterface::class); + $outputMock + ->expects($this->any()) + ->method('writeln') + ->with( + $this->callback(function ($subject) use (&$found, &$fixed) { + if (str_contains($subject, "- Found")) { + $found++; + } elseif (str_contains($subject, "- Fixed")) { + $fixed++; + } + return true; + } + )); + $outputMock + ->expects($this->any()) + ->method('getFormatter') + ->willReturn($this->createMock(OutputFormatterInterface::class)); + + $this->repairMtime->run($this->inputDryRunMock, $outputMock); + $this->assertEquals($found, 10); + $this->assertEquals($fixed, 0); + + $found = 0; + $fixed = 0; + $this->repairMtime->run($this->inputMock, $outputMock); + $this->assertEquals($found, 0); + $this->assertEquals($fixed, 10); + + $found = 0; + $fixed = 0; + $this->repairMtime->run($this->inputDryRunMock, $outputMock); + $this->assertEquals($found, 0); + $this->assertEquals($fixed, 0); + } +} diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index cfd78689fa4dc..ebc6af49a3982 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -148,7 +148,7 @@ private function invalidateCache($key) { * @param $key * @return array|false */ - private function headObject($key) { + public function headObject(string $key) { if (!isset($this->objectCache[$key])) { try { $this->objectCache[$key] = $this->getConnection()->headObject([ diff --git a/lib/private/Files/ObjectStore/Azure.php b/lib/private/Files/ObjectStore/Azure.php index 553f593b29919..811511ad29a27 100644 --- a/lib/private/Files/ObjectStore/Azure.php +++ b/lib/private/Files/ObjectStore/Azure.php @@ -133,4 +133,10 @@ public function objectExists($urn) { public function copyObject($from, $to) { $this->getBlobClient()->copyBlob($this->containerName, $to, $this->containerName, $from); } + + public function headObject(string $urn) { + return $this->getBlobClient() + ->getBlobMetadata($this->containerName, $urn) + ->getMetadata(); + } } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 4e54a26e98a89..7a9875753836f 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -175,4 +175,12 @@ public function objectExists($urn) { public function copyObject($from, $to) { $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to); } + + public function headObject(string $urn) { + return $this->getConnection() + ->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn, + ])->toArray(); + } } diff --git a/lib/private/Files/ObjectStore/StorageObjectStore.php b/lib/private/Files/ObjectStore/StorageObjectStore.php index 85926be897e1c..b37d57e243791 100644 --- a/lib/private/Files/ObjectStore/StorageObjectStore.php +++ b/lib/private/Files/ObjectStore/StorageObjectStore.php @@ -91,4 +91,8 @@ public function objectExists($urn) { public function copyObject($from, $to) { $this->storage->copy($from, $to); } + + public function headObject(string $urn) { + return []; + } } diff --git a/lib/private/Files/ObjectStore/Swift.php b/lib/private/Files/ObjectStore/Swift.php index a3c8d92f3d274..1289265b33894 100644 --- a/lib/private/Files/ObjectStore/Swift.php +++ b/lib/private/Files/ObjectStore/Swift.php @@ -151,4 +151,10 @@ public function copyObject($from, $to) { 'destination' => $this->getContainer()->name . '/' . $to ]); } + + public function headObject(string $urn) { + return $this->getContainer() + ->getObject($urn) + ->getMetadata(); + } } diff --git a/lib/public/Files/ObjectStore/IObjectStore.php b/lib/public/Files/ObjectStore/IObjectStore.php index 924141a3d4bbf..2d8940b5f370e 100644 --- a/lib/public/Files/ObjectStore/IObjectStore.php +++ b/lib/public/Files/ObjectStore/IObjectStore.php @@ -81,4 +81,11 @@ public function objectExists($urn); * @since 21.0.0 */ public function copyObject($from, $to); + + /** + * @param $urn the unified resource name used to identify the object + * @return array|false + * @since 24.0.0 + */ + public function headObject(string $urn); } diff --git a/tests/lib/Files/ObjectStore/FailDeleteObjectStore.php b/tests/lib/Files/ObjectStore/FailDeleteObjectStore.php index 5160abe574fd3..398b17fea5c78 100644 --- a/tests/lib/Files/ObjectStore/FailDeleteObjectStore.php +++ b/tests/lib/Files/ObjectStore/FailDeleteObjectStore.php @@ -55,4 +55,8 @@ public function objectExists($urn) { public function copyObject($from, $to) { $this->objectStore->copyObject($from, $to); } + + public function headObject(string $urn) { + return $this->objectStore->headObject($urn); + } } diff --git a/tests/lib/Files/ObjectStore/FailWriteObjectStore.php b/tests/lib/Files/ObjectStore/FailWriteObjectStore.php index 559d004cd0cd1..a8d88a2cc92c3 100644 --- a/tests/lib/Files/ObjectStore/FailWriteObjectStore.php +++ b/tests/lib/Files/ObjectStore/FailWriteObjectStore.php @@ -56,4 +56,8 @@ public function objectExists($urn) { public function copyObject($from, $to) { $this->objectStore->copyObject($from, $to); } + + public function headObject(string $urn) { + return $this->objectStore->headObject($urn); + } }