From 02b984debd0f7d6f902716b6572915f2deef1608 Mon Sep 17 00:00:00 2001 From: Anna Larch Date: Wed, 8 May 2024 20:38:51 +0200 Subject: [PATCH 1/3] fix(caldav): automatically delete outdated scheduling objects Signed-off-by: Anna Larch --- .../composer/composer/autoload_classmap.php | 2 + .../dav/composer/composer/autoload_static.php | 2 + .../DeleteOutdatedSchedulingObjects.php | 35 +++++++++++++++ apps/dav/lib/CalDAV/CalDavBackend.php | 38 ++++++++++++++++ .../lib/Migration/DeleteSchedulingObjects.php | 38 ++++++++++++++++ .../Version1004Date20170825134824.php | 1 + .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + .../lib/Controller/CheckSetupController.php | 3 ++ .../lib/SetupChecks/SchedulingTableSize.php | 44 +++++++++++++++++++ core/Application.php | 4 ++ core/Command/Db/AddMissingIndices.php | 12 ++++- core/js/setupchecks.js | 1 + lib/private/Repair.php | 2 + 14 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php create mode 100644 apps/dav/lib/Migration/DeleteSchedulingObjects.php create mode 100644 apps/settings/lib/SetupChecks/SchedulingTableSize.php diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index b8cf972a09c4d..d327a5e3a92bf 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -16,6 +16,7 @@ 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\PruneOutdatedSyncTokensJob' => $baseDir . '/../lib/BackgroundJob/PruneOutdatedSyncTokensJob.php', @@ -270,6 +271,7 @@ 'OCA\\DAV\\Migration\\BuildSocialSearchIndexBackgroundJob' => $baseDir . '/../lib/Migration/BuildSocialSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => $baseDir . '/../lib/Migration/CalDAVRemoveEmptyValue.php', 'OCA\\DAV\\Migration\\ChunkCleanup' => $baseDir . '/../lib/Migration/ChunkCleanup.php', + 'OCA\\DAV\\Migration\\DeleteSchedulingObjects' => $baseDir . '/../lib/Migration/DeleteSchedulingObjects.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => $baseDir . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => $baseDir . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\RegenerateBirthdayCalendars' => $baseDir . '/../lib/Migration/RegenerateBirthdayCalendars.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index ee586613ba086..5ee7b7d4326d7 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -31,6 +31,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\PruneOutdatedSyncTokensJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/PruneOutdatedSyncTokensJob.php', @@ -285,6 +286,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Migration\\BuildSocialSearchIndexBackgroundJob' => __DIR__ . '/..' . '/../lib/Migration/BuildSocialSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => __DIR__ . '/..' . '/../lib/Migration/CalDAVRemoveEmptyValue.php', 'OCA\\DAV\\Migration\\ChunkCleanup' => __DIR__ . '/..' . '/../lib/Migration/ChunkCleanup.php', + 'OCA\\DAV\\Migration\\DeleteSchedulingObjects' => __DIR__ . '/..' . '/../lib/Migration/DeleteSchedulingObjects.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => __DIR__ . '/..' . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => __DIR__ . '/..' . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\RegenerateBirthdayCalendars' => __DIR__ . '/..' . '/../lib/Migration/RegenerateBirthdayCalendars.php', diff --git a/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php b/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php new file mode 100644 index 0000000000000..fa53a8be4f02a --- /dev/null +++ b/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php @@ -0,0 +1,35 @@ +setInterval(23 * 60 * 60); + $this->setTimeSensitivity(self::TIME_INSENSITIVE); + } + + /** + * @param array $argument + */ + protected function run($argument): void { + $time = $this->time->getTime() - (60 * 60); + $this->calDavBackend->deleteOutdatedSchedulingObjects($time, 50000); + $this->logger->info("Removed outdated scheduling objects"); + } +} diff --git a/apps/dav/lib/CalDAV/CalDavBackend.php b/apps/dav/lib/CalDAV/CalDavBackend.php index ea332af3933fe..bae9427b655ec 100644 --- a/apps/dav/lib/CalDAV/CalDavBackend.php +++ b/apps/dav/lib/CalDAV/CalDavBackend.php @@ -2722,6 +2722,44 @@ public function deleteSchedulingObject($principalUri, $objectUri) { ->executeStatement(); } + /** + * Deletes all scheduling objects last modified before $modifiedBefore from the inbox collection. + * + * @param int $modifiedBefore + * @param int $limit + * @return void + */ + public function deleteOutdatedSchedulingObjects(int $modifiedBefore, int $limit): void { + $query = $this->db->getQueryBuilder(); + $query->select('id') + ->from('schedulingobjects') + ->where($query->expr()->lt('lastmodified', $query->createNamedParameter($modifiedBefore))) + ->setMaxResults($limit); + $result = $query->executeQuery(); + $count = $result->rowCount(); + if($count === 0) { + return; + } + $ids = array_map(static function (array $id) { + return (int)$id[0]; + }, $result->fetchAll(\PDO::FETCH_NUM)); + $result->closeCursor(); + + $numDeleted = 0; + $deleteQuery = $this->db->getQueryBuilder(); + $deleteQuery->delete('schedulingobjects') + ->where($deleteQuery->expr()->in('id', $deleteQuery->createParameter('ids'), IQueryBuilder::PARAM_INT_ARRAY)); + foreach(array_chunk($ids, 1000) as $chunk) { + $deleteQuery->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $numDeleted += $deleteQuery->executeStatement(); + } + + if($numDeleted === $limit) { + $this->logger->info("Deleted $limit scheduling objects, continuing with next batch"); + $this->deleteOutdatedSchedulingObjects($modifiedBefore, $limit); + } + } + /** * Creates a new scheduling object. This should land in a users' inbox. * diff --git a/apps/dav/lib/Migration/DeleteSchedulingObjects.php b/apps/dav/lib/Migration/DeleteSchedulingObjects.php new file mode 100644 index 0000000000000..3919236788bb3 --- /dev/null +++ b/apps/dav/lib/Migration/DeleteSchedulingObjects.php @@ -0,0 +1,38 @@ +info('Cleaning up old scheduling events'); + $time = $this->time->getTime() - (60 * 60); + $this->calDavBackend->deleteOutdatedSchedulingObjects($time, 50000); + if (!$this->jobList->has(DeleteOutdatedSchedulingObjects::class, null)) { + $output->info('Adding background job to delete old scheduling objects'); + $this->jobList->add(DeleteOutdatedSchedulingObjects::class, null); + } + } +} diff --git a/apps/dav/lib/Migration/Version1004Date20170825134824.php b/apps/dav/lib/Migration/Version1004Date20170825134824.php index a7cbaa78ef23e..7321bba62ff92 100644 --- a/apps/dav/lib/Migration/Version1004Date20170825134824.php +++ b/apps/dav/lib/Migration/Version1004Date20170825134824.php @@ -383,6 +383,7 @@ public function changeSchema(IOutput $output, \Closure $schemaClosure, array $op ]); $table->setPrimaryKey(['id']); $table->addIndex(['principaluri'], 'schedulobj_principuri_index'); + $table->addIndex(['lastmodified'], 'schedulobj_lastmodified_idx'); } if (!$schema->hasTable('cards_properties')) { diff --git a/apps/settings/composer/composer/autoload_classmap.php b/apps/settings/composer/composer/autoload_classmap.php index a1e0b7632ca5f..8f89ada8994ea 100644 --- a/apps/settings/composer/composer/autoload_classmap.php +++ b/apps/settings/composer/composer/autoload_classmap.php @@ -79,6 +79,7 @@ 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync' => $baseDir . '/../lib/SetupChecks/NeedsSystemAddressBookSync.php', 'OCA\\Settings\\SetupChecks\\PhpDefaultCharset' => $baseDir . '/../lib/SetupChecks/PhpDefaultCharset.php', 'OCA\\Settings\\SetupChecks\\PhpOutputBuffering' => $baseDir . '/../lib/SetupChecks/PhpOutputBuffering.php', + 'OCA\\Settings\\SetupChecks\\SchedulingTableSize' => $baseDir . '/../lib/SetupChecks/SchedulingTableSize.php', 'OCA\\Settings\\SetupChecks\\SupportedDatabase' => $baseDir . '/../lib/SetupChecks/SupportedDatabase.php', 'OCA\\Settings\\UserMigration\\AccountMigrator' => $baseDir . '/../lib/UserMigration/AccountMigrator.php', 'OCA\\Settings\\UserMigration\\AccountMigratorException' => $baseDir . '/../lib/UserMigration/AccountMigratorException.php', diff --git a/apps/settings/composer/composer/autoload_static.php b/apps/settings/composer/composer/autoload_static.php index 1eef91c5d91cf..77ec7c0fb612d 100644 --- a/apps/settings/composer/composer/autoload_static.php +++ b/apps/settings/composer/composer/autoload_static.php @@ -94,6 +94,7 @@ class ComposerStaticInitSettings 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync' => __DIR__ . '/..' . '/../lib/SetupChecks/NeedsSystemAddressBookSync.php', 'OCA\\Settings\\SetupChecks\\PhpDefaultCharset' => __DIR__ . '/..' . '/../lib/SetupChecks/PhpDefaultCharset.php', 'OCA\\Settings\\SetupChecks\\PhpOutputBuffering' => __DIR__ . '/..' . '/../lib/SetupChecks/PhpOutputBuffering.php', + 'OCA\\Settings\\SetupChecks\\SchedulingTableSize' => __DIR__ . '/..' . '/../lib/SetupChecks/SchedulingTableSize.php', 'OCA\\Settings\\SetupChecks\\SupportedDatabase' => __DIR__ . '/..' . '/../lib/SetupChecks/SupportedDatabase.php', 'OCA\\Settings\\UserMigration\\AccountMigrator' => __DIR__ . '/..' . '/../lib/UserMigration/AccountMigrator.php', 'OCA\\Settings\\UserMigration\\AccountMigratorException' => __DIR__ . '/..' . '/../lib/UserMigration/AccountMigratorException.php', diff --git a/apps/settings/lib/Controller/CheckSetupController.php b/apps/settings/lib/Controller/CheckSetupController.php index bf8de16a2b9b5..4de8e3c23896e 100644 --- a/apps/settings/lib/Controller/CheckSetupController.php +++ b/apps/settings/lib/Controller/CheckSetupController.php @@ -68,6 +68,7 @@ use OCA\Settings\SetupChecks\PhpDefaultCharset; use OCA\Settings\SetupChecks\PhpOutputBuffering; use OCA\Settings\SetupChecks\SupportedDatabase; +use OCA\Settings\SetupChecks\SchedulingTableSize; use OCP\App\IAppManager; use OCP\AppFramework\Controller; use OCP\AppFramework\Http\DataDisplayResponse; @@ -902,6 +903,7 @@ public function check() { $supportedDatabases = new SupportedDatabase($this->l10n, $this->connection); $ldapInvalidUuids = new LdapInvalidUuids($this->appManager, $this->l10n, $this->serverContainer); $needsSystemAddressBookSync = new NeedsSystemAddressBookSync($this->config, $this->l10n); + $schedulingTableSize = new SchedulingTableSize($this->l10n, $this->connection); return new DataResponse( [ @@ -958,6 +960,7 @@ public function check() { 'temporaryDirectoryWritable' => $this->isTemporaryDirectoryWritable(), LdapInvalidUuids::class => ['pass' => $ldapInvalidUuids->run(), 'description' => $ldapInvalidUuids->description(), 'severity' => $ldapInvalidUuids->severity()], NeedsSystemAddressBookSync::class => ['pass' => $needsSystemAddressBookSync->run(), 'description' => $needsSystemAddressBookSync->description(), 'severity' => $needsSystemAddressBookSync->severity()], + SchedulingTableSize::class => ['pass' => $schedulingTableSize->run(), 'description' => $schedulingTableSize->description(), 'severity' => $schedulingTableSize->severity()], ] ); } diff --git a/apps/settings/lib/SetupChecks/SchedulingTableSize.php b/apps/settings/lib/SetupChecks/SchedulingTableSize.php new file mode 100644 index 0000000000000..ea1908215bffa --- /dev/null +++ b/apps/settings/lib/SetupChecks/SchedulingTableSize.php @@ -0,0 +1,44 @@ +l10n = $l10n; + $this->connection = $connection; + } + + public function description(): string { + return $this->l10n->t('You have more than 500 000 rows in the scheduling objects table. Please run the expensive repair jobs via occ maintenance:repair --include-expensive'); + } + + public function severity(): string { + return 'warning'; + } + + public function run(): bool { + $qb = $this->connection->getQueryBuilder(); + $qb->select($qb->func()->count('id')) + ->from('schedulingobjects'); + $query = $qb->executeQuery(); + $count = $query->fetchOne(); + $query->closeCursor(); + + return $count <= 500000; + } +} diff --git a/core/Application.php b/core/Application.php index 592e092966681..ee77e447b03ae 100644 --- a/core/Application.php +++ b/core/Application.php @@ -201,6 +201,10 @@ function (GenericEvent $event) use ($container) { if (!$table->hasIndex('schedulobj_principuri_index')) { $subject->addHintForMissingSubject($table->getName(), 'schedulobj_principuri_index'); } + $table = $schema->getTable('schedulingobjects'); + if (!$table->hasIndex('schedulobj_lastmodified_idx')) { + $subject->addHintForMissingSubject($table->getName(), 'schedulobj_lastmodified_idx'); + } } if ($schema->hasTable('properties')) { diff --git a/core/Command/Db/AddMissingIndices.php b/core/Command/Db/AddMissingIndices.php index 1044cea554780..78cde82ef8a5d 100644 --- a/core/Command/Db/AddMissingIndices.php +++ b/core/Command/Db/AddMissingIndices.php @@ -403,8 +403,18 @@ private function addCoreIndexes(OutputInterface $output, bool $dryRun): void { $output->writeln($sqlQueries); } $updated = true; - $output->writeln('schedulingobjects table updated successfully.'); } + if (!$table->hasIndex('schedulobj_lastmodified_idx')) { + $output->writeln('Adding schedulobj_lastmodified_idx index to the schedulingobjects table, this can take some time...'); + + $table->addIndex(['lastmodified'], 'schedulobj_lastmodified_idx'); + $sqlQueries = $this->connection->migrateToSchema($schema->getWrappedSchema(), $dryRun); + if ($dryRun && $sqlQueries !== null) { + $output->writeln($sqlQueries); + } + $updated = true; + } + $output->writeln('schedulingobjects table updated successfully.'); } $output->writeln('Check indices of the oc_properties table.'); diff --git a/core/js/setupchecks.js b/core/js/setupchecks.js index 3a836cd78c6d0..613aae1f084ce 100644 --- a/core/js/setupchecks.js +++ b/core/js/setupchecks.js @@ -543,6 +543,7 @@ OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\SupportedDatabase', messages) OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\LdapInvalidUuids', messages) OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync', messages) + OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\SchedulingTableSize', messages) } else { messages.push({ msg: t('core', 'Error occurred while checking server setup'), diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 99af19ddb79f4..04f7981db9639 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -81,6 +81,7 @@ use OC\Repair\RepairMimeTypes; use OC\Repair\SqliteAutoincrement; use OC\Template\JSCombiner; +use OCA\DAV\Migration\DeleteSchedulingObjects; use OCP\AppFramework\QueryException; use OCP\AppFramework\Utility\ITimeFactory; use OCP\Collaboration\Resources\IManager; @@ -227,6 +228,7 @@ public static function getExpensiveRepairSteps() { return [ new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()), \OC::$server->get(ValidatePhoneNumber::class), + \OC::$server->get(DeleteSchedulingObjects::class), ]; } From 674d48bddd782558f426f7135f320600e5df7be2 Mon Sep 17 00:00:00 2001 From: Anna Larch Date: Tue, 4 Jun 2024 11:20:59 +0200 Subject: [PATCH 2/3] fixup! fix(caldav): automatically delete outdated scheduling objects --- apps/settings/tests/Controller/CheckSetupControllerTest.php | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/settings/tests/Controller/CheckSetupControllerTest.php b/apps/settings/tests/Controller/CheckSetupControllerTest.php index 50723703c5672..8261f322b5378 100644 --- a/apps/settings/tests/Controller/CheckSetupControllerTest.php +++ b/apps/settings/tests/Controller/CheckSetupControllerTest.php @@ -671,6 +671,7 @@ public function testCheck() { 'temporaryDirectoryWritable' => false, \OCA\Settings\SetupChecks\LdapInvalidUuids::class => ['pass' => true, 'description' => 'Invalid UUIDs of LDAP users or groups have been found. Please review your "Override UUID detection" settings in the Expert part of the LDAP configuration and use "occ ldap:update-uuid" to update them.', 'severity' => 'warning'], \OCA\Settings\SetupChecks\NeedsSystemAddressBookSync::class => ['pass' => true, 'description' => 'The DAV system address book sync has not run yet as your instance has more than 1000 users or because an error occured. Please run it manually by calling occ dav:sync-system-addressbook.', 'severity' => 'warning'], + \OCA\Settings\SetupChecks\SchedulingTableSize::class => ['pass' => true, 'description' => 'You have more than 500 000 rows in the scheduling objects table. Please run the expensive repair jobs via occ maintenance:repair --include-expensive', 'severity' => 'warning'], 'isBruteforceThrottled' => false, 'bruteforceRemoteAddress' => '', ] From 602496ec8747429788c7a6ee4f8330b5909cf2d9 Mon Sep 17 00:00:00 2001 From: Anna Larch Date: Tue, 4 Jun 2024 12:17:23 +0200 Subject: [PATCH 3/3] fixup! fix(caldav): automatically delete outdated scheduling objects --- apps/settings/tests/Controller/CheckSetupControllerTest.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/settings/tests/Controller/CheckSetupControllerTest.php b/apps/settings/tests/Controller/CheckSetupControllerTest.php index 8261f322b5378..67fd97aaeaaec 100644 --- a/apps/settings/tests/Controller/CheckSetupControllerTest.php +++ b/apps/settings/tests/Controller/CheckSetupControllerTest.php @@ -47,6 +47,7 @@ use OCP\AppFramework\Http\DataDisplayResponse; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\RedirectResponse; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\EventDispatcher\IEventDispatcher; use OCP\Http\Client\IClientService; use OCP\IConfig; @@ -610,6 +611,9 @@ public function testCheck() { $sqlitePlatform = $this->getMockBuilder(SqlitePlatform::class)->getMock(); $this->connection->method('getDatabasePlatform') ->willReturn($sqlitePlatform); + $queryBuilder = $this->getMockBuilder(IQueryBuilder::class)->getMock(); + $this->connection->method('getQueryBuilder') + ->willReturn($queryBuilder); $expected = new DataResponse( [