Skip to content

Commit 4279ff1

Browse files
committed
feat(dav): introduce paginate with custom headers
Signed-off-by: Benjamin Gaussorgues <benjamin.gaussorgues@nextcloud.com>
1 parent d013a13 commit 4279ff1

File tree

9 files changed

+330
-1
lines changed

9 files changed

+330
-1
lines changed

apps/dav/appinfo/info.xml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
<name>WebDAV</name>
1111
<summary>WebDAV endpoint</summary>
1212
<description>WebDAV endpoint</description>
13-
<version>1.32.0</version>
13+
<version>1.33.0</version>
1414
<licence>agpl</licence>
1515
<author>owncloud.org</author>
1616
<namespace>DAV</namespace>
@@ -27,6 +27,7 @@
2727
<job>OCA\DAV\BackgroundJob\CleanupDirectLinksJob</job>
2828
<job>OCA\DAV\BackgroundJob\UpdateCalendarResourcesRoomsBackgroundJob</job>
2929
<job>OCA\DAV\BackgroundJob\CleanupInvitationTokenJob</job>
30+
<job>OCA\DAV\BackgroundJob\CleanupPaginateCacheJob</job>
3031
<job>OCA\DAV\BackgroundJob\EventReminderJob</job>
3132
<job>OCA\DAV\BackgroundJob\CalendarRetentionJob</job>
3233
<job>OCA\DAV\BackgroundJob\PruneOutdatedSyncTokensJob</job>

apps/dav/composer/composer/autoload_classmap.php

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php',
1717
'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
1818
'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
19+
'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => $baseDir . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php',
1920
'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php',
2021
'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php',
2122
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
@@ -327,6 +328,7 @@
327328
'OCA\\DAV\\Migration\\Version1008Date20181105110300' => $baseDir . '/../lib/Migration/Version1008Date20181105110300.php',
328329
'OCA\\DAV\\Migration\\Version1008Date20181105112049' => $baseDir . '/../lib/Migration/Version1008Date20181105112049.php',
329330
'OCA\\DAV\\Migration\\Version1008Date20181114084440' => $baseDir . '/../lib/Migration/Version1008Date20181114084440.php',
331+
'OCA\\DAV\\Migration\\Version1009Date20181108161232' => $baseDir . '/../lib/Migration/Version1009Date20181108161232.php',
330332
'OCA\\DAV\\Migration\\Version1011Date20190725113607' => $baseDir . '/../lib/Migration/Version1011Date20190725113607.php',
331333
'OCA\\DAV\\Migration\\Version1011Date20190806104428' => $baseDir . '/../lib/Migration/Version1011Date20190806104428.php',
332334
'OCA\\DAV\\Migration\\Version1012Date20190808122342' => $baseDir . '/../lib/Migration/Version1012Date20190808122342.php',
@@ -340,6 +342,10 @@
340342
'OCA\\DAV\\Migration\\Version1029Date20231004091403' => $baseDir . '/../lib/Migration/Version1029Date20231004091403.php',
341343
'OCA\\DAV\\Migration\\Version1030Date20240205103243' => $baseDir . '/../lib/Migration/Version1030Date20240205103243.php',
342344
'OCA\\DAV\\Migration\\Version1031Date20240610134258' => $baseDir . '/../lib/Migration/Version1031Date20240610134258.php',
345+
'OCA\\DAV\\Migration\\Version1032Date20241011093632' => $baseDir . '/../lib/Migration/Version1032Date20241011093632.php',
346+
'OCA\\DAV\\Paginate\\LimitedCopyIterator' => $baseDir . '/../lib/Paginate/LimitedCopyIterator.php',
347+
'OCA\\DAV\\Paginate\\PaginateCache' => $baseDir . '/../lib/Paginate/PaginateCache.php',
348+
'OCA\\DAV\\Paginate\\PaginatePlugin' => $baseDir . '/../lib/Paginate/PaginatePlugin.php',
343349
'OCA\\DAV\\Profiler\\ProfilerPlugin' => $baseDir . '/../lib/Profiler/ProfilerPlugin.php',
344350
'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
345351
'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',

apps/dav/composer/composer/autoload_static.php

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class ComposerStaticInitDAV
3131
'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php',
3232
'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
3333
'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
34+
'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php',
3435
'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php',
3536
'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php',
3637
'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
@@ -342,6 +343,7 @@ class ComposerStaticInitDAV
342343
'OCA\\DAV\\Migration\\Version1008Date20181105110300' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105110300.php',
343344
'OCA\\DAV\\Migration\\Version1008Date20181105112049' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105112049.php',
344345
'OCA\\DAV\\Migration\\Version1008Date20181114084440' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181114084440.php',
346+
'OCA\\DAV\\Migration\\Version1009Date20181108161232' => __DIR__ . '/..' . '/../lib/Migration/Version1009Date20181108161232.php',
345347
'OCA\\DAV\\Migration\\Version1011Date20190725113607' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190725113607.php',
346348
'OCA\\DAV\\Migration\\Version1011Date20190806104428' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190806104428.php',
347349
'OCA\\DAV\\Migration\\Version1012Date20190808122342' => __DIR__ . '/..' . '/../lib/Migration/Version1012Date20190808122342.php',
@@ -355,6 +357,10 @@ class ComposerStaticInitDAV
355357
'OCA\\DAV\\Migration\\Version1029Date20231004091403' => __DIR__ . '/..' . '/../lib/Migration/Version1029Date20231004091403.php',
356358
'OCA\\DAV\\Migration\\Version1030Date20240205103243' => __DIR__ . '/..' . '/../lib/Migration/Version1030Date20240205103243.php',
357359
'OCA\\DAV\\Migration\\Version1031Date20240610134258' => __DIR__ . '/..' . '/../lib/Migration/Version1031Date20240610134258.php',
360+
'OCA\\DAV\\Migration\\Version1032Date20241011093632' => __DIR__ . '/..' . '/../lib/Migration/Version1032Date20241011093632.php',
361+
'OCA\\DAV\\Paginate\\LimitedCopyIterator' => __DIR__ . '/..' . '/../lib/Paginate/LimitedCopyIterator.php',
362+
'OCA\\DAV\\Paginate\\PaginateCache' => __DIR__ . '/..' . '/../lib/Paginate/PaginateCache.php',
363+
'OCA\\DAV\\Paginate\\PaginatePlugin' => __DIR__ . '/..' . '/../lib/Paginate/PaginatePlugin.php',
358364
'OCA\\DAV\\Profiler\\ProfilerPlugin' => __DIR__ . '/..' . '/../lib/Profiler/ProfilerPlugin.php',
359365
'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
360366
'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\DAV\BackgroundJob;
11+
12+
use OC\BackgroundJob\Job;
13+
use OCA\DAV\Paginate\PaginateCache;
14+
15+
class CleanupPaginateCacheJob extends Job {
16+
17+
/** @var PaginateCache */
18+
private $cache;
19+
20+
public function __construct(PaginateCache $cache) {
21+
$this->cache = $cache;
22+
}
23+
24+
public function run($argument) {
25+
$this->cache->cleanup();
26+
}
27+
28+
}
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\DAV\Migration;
11+
12+
use OCP\DB\ISchemaWrapper;
13+
use OCP\DB\Types;
14+
use OCP\Migration\IOutput;
15+
use OCP\Migration\SimpleMigrationStep;
16+
17+
class Version1032Date20241011093632 extends SimpleMigrationStep {
18+
public function name(): string {
19+
return 'Add dav_page_cache table';
20+
}
21+
22+
public function description(): string {
23+
return 'Add table to cache webdav multistatus responses for pagination purpose';
24+
}
25+
26+
public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) {
27+
/** @var ISchemaWrapper $schema */
28+
$schema = $schemaClosure();
29+
30+
if (!$schema->hasTable('dav_page_cache')) {
31+
$table = $schema->createTable('dav_page_cache');
32+
33+
$table->addColumn('id', Types::BIGINT, [
34+
'autoincrement' => true
35+
]);
36+
$table->addColumn('url_hash', Types::STRING, [
37+
'notnull' => true,
38+
'length' => 32,
39+
]);
40+
$table->addColumn('token', Types::STRING, [
41+
'notnull' => true,
42+
'length' => 32
43+
]);
44+
$table->addColumn('result_index', Types::INTEGER, [
45+
'notnull' => true
46+
]);
47+
$table->addColumn('result_value', Types::TEXT, [
48+
'notnull' => false,
49+
]);
50+
$table->addColumn('insert_time', Types::DATETIME, [
51+
'notnull' => true,
52+
]);
53+
54+
$table->setPrimaryKey(['id'], 'dav_page_cache_id_index');
55+
$table->addIndex(['token', 'url_hash'], 'dav_page_cache_token_url');
56+
$table->addUniqueIndex(['token', 'url_hash', 'result_index'], 'dav_page_cache_url_index');
57+
$table->addIndex(['result_index'], 'dav_page_cache_index');
58+
$table->addIndex(['insert_time'], 'dav_page_cache_time');
59+
}
60+
61+
return $schema;
62+
}
63+
}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\DAV\Paginate;
11+
12+
/**
13+
* Save a copy of the first X items into a separate iterator
14+
*
15+
* this allows us to pass the iterator to the cache while keeping a copy
16+
* of the first X items
17+
*/
18+
class LimitedCopyIterator extends \AppendIterator {
19+
/** @var array */
20+
private array $copy = [];
21+
22+
public function __construct(\Traversable $iterator, int $count) {
23+
parent::__construct();
24+
25+
if (!$iterator instanceof \Iterator) {
26+
$iterator = new \IteratorIterator($iterator);
27+
}
28+
$iterator = new \NoRewindIterator($iterator);
29+
30+
while ($iterator->valid() && count($this->copy) < $count) {
31+
$this->copy[] = $iterator->current();
32+
$iterator->next();
33+
}
34+
35+
$this->append($this->getFirstItems());
36+
$this->append($iterator);
37+
}
38+
39+
public function getFirstItems(): \Iterator {
40+
return new \ArrayIterator($this->copy);
41+
}
42+
}
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCA\DAV\Paginate;
11+
12+
use OCP\AppFramework\Utility\ITimeFactory;
13+
use OCP\DB\QueryBuilder\IQueryBuilder;
14+
use OCP\IDBConnection;
15+
use OCP\Security\ISecureRandom;
16+
17+
class PaginateCache {
18+
public const TTL = 3600;
19+
20+
public function __construct(
21+
private IDBConnection $database,
22+
private ISecureRandom $random,
23+
private ITimeFactory $timeFactory,
24+
) {
25+
}
26+
27+
public function store(string $uri, \Iterator $items): array {
28+
$token = $this->random->generate(32);
29+
$now = $this->timeFactory->getTime();
30+
31+
$query = $this->database->getQueryBuilder();
32+
$query->insert('dav_page_cache')
33+
->values([
34+
'url_hash' => $query->createNamedParameter(md5($uri), IQueryBuilder::PARAM_STR),
35+
'token' => $query->createNamedParameter($token, IQueryBuilder::PARAM_STR),
36+
'insert_time' => $query->createNamedParameter($now, IQueryBuilder::PARAM_INT),
37+
'result_index' => $query->createParameter('index'),
38+
'result_value' => $query->createParameter('value'),
39+
]);
40+
41+
$count = 0;
42+
foreach ($items as $item) {
43+
$value = json_encode($item);
44+
$query->setParameter('index', $count, IQueryBuilder::PARAM_INT);
45+
$query->setParameter('value', $value);
46+
$query->executeStatement();
47+
$count++;
48+
}
49+
50+
return [$token, $count];
51+
}
52+
53+
/**
54+
* @param string $url
55+
* @param string $token
56+
* @param int $offset
57+
* @param int $count
58+
* @return array|\Traversable
59+
*/
60+
public function get(string $url, string $token, int $offset, int $count) {
61+
$query = $this->database->getQueryBuilder();
62+
$query->select(['result_value'])
63+
->from('dav_page_cache')
64+
->where($query->expr()->eq('token', $query->createNamedParameter($token)))
65+
->andWhere($query->expr()->eq('url_hash', $query->createNamedParameter(md5($url))))
66+
->andWhere($query->expr()->gte('result_index', $query->createNamedParameter($offset, IQueryBuilder::PARAM_INT)))
67+
->andWhere($query->expr()->lt('result_index', $query->createNamedParameter($offset + $count, IQueryBuilder::PARAM_INT)));
68+
69+
$result = $query->executeQuery();
70+
return array_map(function (string $entry) {
71+
return json_decode($entry, true);
72+
}, $result->fetchAll(\PDO::FETCH_COLUMN));
73+
}
74+
75+
public function cleanup(): void {
76+
$now = $this->timeFactory->getTime();
77+
78+
$query = $this->database->getQueryBuilder();
79+
$query->delete('dav_page_cache')
80+
->where($query->expr()->lt('insert_time', $query->createNamedParameter($now - self::TTL)));
81+
$query->executeStatement();
82+
}
83+
84+
public function clear(): void {
85+
$query = $this->database->getQueryBuilder();
86+
$query->delete('dav_page_cache');
87+
$query->executeStatement();
88+
}
89+
}
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OCA\DAV\Paginate;
11+
12+
use Sabre\DAV\Server;
13+
use Sabre\DAV\ServerPlugin;
14+
use Sabre\HTTP\RequestInterface;
15+
use Sabre\HTTP\ResponseInterface;
16+
17+
class PaginatePlugin extends ServerPlugin {
18+
public const PAGINATE_HEADER = 'x-nc-paginate';
19+
public const PAGINATE_TOTAL_HEADER = 'x-nc-paginate-total';
20+
public const PAGINATE_TOKEN_HEADER = 'x-nc-paginate-token';
21+
public const PAGINATE_OFFSET_HEADER = 'x-nc-paginate-offset';
22+
public const PAGINATE_COUNT_HEADER = 'x-nc-paginate-count';
23+
24+
/** @var Server */
25+
private $server;
26+
27+
public function __construct(
28+
private PaginateCache $cache,
29+
private int $pageSize = 100,
30+
) {
31+
}
32+
33+
public function initialize(Server $server): void {
34+
$this->server = $server;
35+
$server->on('beforeMultiStatus', [$this, 'onMultiStatus']);
36+
$server->on('method:SEARCH', [$this, 'onMethod'], 1);
37+
$server->on('method:PROPFIND', [$this, 'onMethod'], 1);
38+
$server->on('method:REPORT', [$this, 'onMethod'], 1);
39+
}
40+
41+
public function getFeatures(): array {
42+
return ['nc-paginate'];
43+
}
44+
45+
public function onMultiStatus(&$fileProperties): void {
46+
$request = $this->server->httpRequest;
47+
if (is_array($fileProperties)) {
48+
$fileProperties = new \ArrayIterator($fileProperties);
49+
}
50+
if (
51+
$request->hasHeader(self::PAGINATE_HEADER) &&
52+
!$request->hasHeader(self::PAGINATE_TOKEN_HEADER)
53+
) {
54+
$url = $request->getUrl();
55+
56+
$pageSize = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
57+
$copyIterator = new LimitedCopyIterator($fileProperties, $pageSize);
58+
[$token, $count] = $this->cache->store($url, $copyIterator);
59+
60+
$fileProperties = $copyIterator->getFirstItems();
61+
$this->server->httpResponse->addHeader(self::PAGINATE_HEADER, 'true');
62+
$this->server->httpResponse->addHeader(self::PAGINATE_TOKEN_HEADER, $token);
63+
$this->server->httpResponse->addHeader(self::PAGINATE_TOTAL_HEADER, $count);
64+
}
65+
}
66+
67+
public function onMethod(RequestInterface $request, ResponseInterface $response) {
68+
if (
69+
$request->hasHeader(self::PAGINATE_TOKEN_HEADER) &&
70+
$request->hasHeader(self::PAGINATE_OFFSET_HEADER)
71+
) {
72+
$url = $this->server->httpRequest->getUrl();
73+
$token = $request->getHeader(self::PAGINATE_TOKEN_HEADER);
74+
$offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER);
75+
$count = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
76+
77+
$items = $this->cache->get($url, $token, $offset, $count);
78+
79+
$response->setStatus(207);
80+
$response->addHeader(self::PAGINATE_HEADER, 'true');
81+
$response->setHeader('Content-Type', 'application/xml; charset=utf-8');
82+
$response->setHeader('Vary', 'Brief,Prefer');
83+
84+
$prefer = $this->server->getHTTPPrefer();
85+
$minimal = $prefer['return'] === 'minimal';
86+
87+
$data = $this->server->generateMultiStatus($items, $minimal);
88+
$response->setBody($data);
89+
return false;
90+
}
91+
}
92+
}

0 commit comments

Comments
 (0)