Skip to content

Commit 921b8ae

Browse files
andris9andris9
authored
Merge pull request #460 from postalsys/ms-graph-search
Allow using $search instead of $filter when searching MS Graph API accounts
2 parents 8fe9dc8 + 2d455c9 commit 921b8ae

File tree

11 files changed

+225
-104
lines changed

11 files changed

+225
-104
lines changed

‎data/google-crawlers.json‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"creationTime": "2024-10-15T14:49:32.000000",
2+
"creationTime": "2024-10-22T14:46:44.000000",
33
"prefixes": [
44
{
55
"ipv6Prefix": "2001:4860:4801:2008::/64"
@@ -181,6 +181,9 @@
181181
{
182182
"ipv6Prefix": "2001:4860:4801:204a::/64"
183183
},
184+
{
185+
"ipv6Prefix": "2001:4860:4801:204b::/64"
186+
},
184187
{
185188
"ipv6Prefix": "2001:4860:4801:2050::/64"
186189
},
@@ -313,6 +316,9 @@
313316
{
314317
"ipv6Prefix": "2001:4860:4801:2093::/64"
315318
},
319+
{
320+
"ipv6Prefix": "2001:4860:4801:2094::/64"
321+
},
316322
{
317323
"ipv4Prefix": "108.177.2.0/27"
318324
},
@@ -406,6 +412,9 @@
406412
{
407413
"ipv4Prefix": "66.249.90.32/27"
408414
},
415+
{
416+
"ipv4Prefix": "66.249.90.64/27"
417+
},
409418
{
410419
"ipv4Prefix": "66.249.90.96/27"
411420
},
@@ -532,6 +541,9 @@
532541
{
533542
"ipv4Prefix": "74.125.150.64/27"
534543
},
544+
{
545+
"ipv4Prefix": "74.125.150.96/27"
546+
},
535547
{
536548
"ipv4Prefix": "74.125.151.0/27"
537549
},

‎lib/db.js‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ const fs = require('fs');
99
const config = require('wild-config');
1010
const pathlib = require('path');
1111
const redisUrl = require('./redis-url');
12-
const packageData = require('../package.json');
13-
const { threadId } = require('worker_threads');
12+
//const packageData = require('../package.json');
13+
//const { threadId } = require('worker_threads');
1414
const logger = require('./logger');
1515
const { REDIS_PREFIX } = require('./consts');
1616
const Path = require('path');
@@ -57,8 +57,9 @@ const REDIS_CONF = Object.assign(
5757
logger.fatal({ msg: 'Redis connection error', err });
5858
// always try to reconnect
5959
return true;
60-
},
61-
connectionName: `${packageData.name}@${packageData.version}[${process.pid}${threadId ? `:${threadId}` : ''}]`
60+
}
61+
// Setting connection name triggers CLIENT.SETNAME command which is not supported by many managed hosts
62+
//connectionName: `${packageData.name}@${packageData.version}[${process.pid}${threadId ? `:${threadId}` : ''}]`
6263
},
6364
typeof redisConf === 'string' ? redisUrl(redisConf) : redisConf || {}
6465
);

‎lib/email-client/outlook-client.js‎

Lines changed: 121 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,16 @@ class OutlookClient extends BaseClient {
9292

9393
result = await this.oAuth2Client.request(accessToken, apiUrl, method, payload, options);
9494
} catch (err) {
95+
switch (err.oauthRequest?.response?.error?.code) {
96+
case 'ErrorExecuteSearchStaleData': {
97+
this.logger.error({ msg: 'Invalid or expired paging cursor', account: this.account, err });
98+
let error = new Error('Invalid or expired paging cursor');
99+
error.code = 'InvalidPagingCursor';
100+
error.statusCode = err.oauthRequest?.status || 500;
101+
throw error;
102+
}
103+
}
104+
95105
switch (err.oauthRequest?.status) {
96106
case 401:
97107
this.logger.error({ msg: 'Failed to authenticate API request', account: this.account, accessToken, err });
@@ -326,18 +336,23 @@ class OutlookClient extends BaseClient {
326336

327337
let page = Number(query.page) || 0;
328338
let pageSize = Math.abs(Number(query.pageSize) || 20);
339+
let $skiptoken;
329340

330341
if (query.cursor) {
331-
let cursorPage = this.decodeCursorStr(query.cursor);
342+
let { cursorPage, skipToken } = this.decodeCursorStr(query.cursor);
332343
if (typeof cursorPage === 'number' && cursorPage >= 0) {
333344
page = cursorPage;
334345
}
346+
if (skipToken) {
347+
$skiptoken = skipToken;
348+
}
335349
}
336350

337351
let requestQuery = {
338352
$count: true,
339353
$top: pageSize,
340354
$skip: page * pageSize,
355+
$skiptoken,
341356
$orderBy: 'receivedDateTime desc',
342357
$select: (options.metadataOnly
343358
? ['id', 'conversationId', 'receivedDateTime', 'isRead', 'isDraft', 'flag', 'body', 'subject', 'from', 'replyTo', 'sender', 'internetMessageId']
@@ -365,15 +380,26 @@ class OutlookClient extends BaseClient {
365380
$expand: options.metadataOnly ? undefined : 'attachments($select=id,name,contentType,size,isInline,microsoft.graph.fileAttachment/contentId)'
366381
};
367382

368-
if (query.search) {
369-
const { $search, $filter } = this.prepareQuery(query.search);
370-
if ($search) {
371-
requestQuery.$search = `"${$search}"`;
372-
}
383+
let useOutlookSearch = false;
384+
let skipToken = null;
373385

374-
if ($filter) {
375-
// we need to have receivedDateTime as the first filtering property, otherwise ordering will fail
376-
requestQuery.$filter = `receivedDateTime gt 1970-01-01T00:00:00.000Z and ${$filter}`;
386+
if (query.search) {
387+
if (query.useOutlookSearch) {
388+
const $search = this.prepareSearchQuery(query.search);
389+
if ($search) {
390+
requestQuery.$search = `"${$search}"`;
391+
// remove unsupported request arguments
392+
for (let disabledParam of ['$skip', '$orderBy', '$count']) {
393+
delete requestQuery[disabledParam];
394+
}
395+
useOutlookSearch = true;
396+
}
397+
} else {
398+
const $filter = this.prepareFilterQuery(query.search);
399+
if ($filter) {
400+
// we need to have receivedDateTime as the first filtering property, otherwise ordering will fail
401+
requestQuery.$filter = `receivedDateTime gt 1970-01-01T00:00:00.000Z and ${$filter}`;
402+
}
377403
}
378404
}
379405

@@ -384,7 +410,12 @@ class OutlookClient extends BaseClient {
384410
try {
385411
let listing = await this.request(`/me/${folder ? `mailFolders/${folder.id}/` : ''}messages`, 'get', requestQuery);
386412

387-
totalMessages = !isNaN(listing['@odata.count']) ? Number(listing['@odata.count']) : 0;
413+
totalMessages = !isNaN(listing['@odata.count']) ? Number(listing['@odata.count']) : undefined;
414+
415+
if (useOutlookSearch && listing['@odata.nextLink']) {
416+
let nextLinkObj = new URL(listing['@odata.nextLink']);
417+
skipToken = nextLinkObj.searchParams.get('$skiptoken');
418+
}
388419

389420
messages =
390421
listing?.value?.map(messageData =>
@@ -400,14 +431,15 @@ class OutlookClient extends BaseClient {
400431
throw err;
401432
}
402433

403-
let pages = Math.ceil(totalMessages / pageSize) || 1;
434+
let pages = typeof totalMessages === 'number' ? Math.ceil(totalMessages / pageSize) || 1 : undefined;
404435

405436
if (page < 0) {
406437
page = 0;
407438
}
408439

409-
let nextPageCursor = page < pages - 1 ? this.encodeCursorString(page + 1) : null;
410-
let prevPageCursor = page > 0 ? this.encodeCursorString(Math.min(page - 1, pages - 1)) : null;
440+
let nextPageCursor = page < pages - 1 || skipToken ? this.encodeCursorString(page + 1, skipToken) : null;
441+
// no previous page cursor if we are using skip token for paging
442+
let prevPageCursor = skipToken ? undefined : page > 0 ? this.encodeCursorString(Math.min(page - 1, pages - 1)) : null;
411443

412444
return {
413445
total: totalMessages,
@@ -2321,6 +2353,10 @@ class OutlookClient extends BaseClient {
23212353
return;
23222354
}
23232355

2356+
// check if we have seen this message before or not (approximate estimation, not 100% exact)
2357+
messageData.seemsLikeNew =
2358+
messageData.messageSpecialUse !== '\\Sent' && !!(await this.connection.redis.pfadd(this.getSeenMessagesKey(), messageData.messageId));
2359+
23242360
return messageData;
23252361
}
23262362

@@ -2361,8 +2397,68 @@ class OutlookClient extends BaseClient {
23612397
return messages.map(message => message.id);
23622398
}
23632399

2364-
// convert IMAP SEARCH query object to a Gmail API search query
2365-
prepareQuery(search) {
2400+
// convert IMAP SEARCH query object to a $search query
2401+
prepareSearchQuery(search) {
2402+
search = search || {};
2403+
2404+
const searchParts = [];
2405+
2406+
const enabledKeys = ['to', 'cc', 'bcc', 'larger', 'smaller', 'body', 'before', 'sentBefore', 'since', 'sentSince'];
2407+
2408+
// not supported search terms
2409+
for (let key of Object.keys(search)) {
2410+
if (!enabledKeys.includes(key)) {
2411+
let error = new Error(`Unsupported search term "${key}" for Outlook Search`);
2412+
error.code = 'UnsupportedSearchTerm';
2413+
error.statusCode = 400;
2414+
throw error;
2415+
}
2416+
}
2417+
2418+
let escapeString = term => {
2419+
if (typeof term === 'object' && term && Object.prototype.toString.apply(new Date()) === '[object Date]') {
2420+
// convert dates to "MM/DD/YYYY"
2421+
let d = term.getDate();
2422+
let m = term.getMonth() + 1;
2423+
let y = term.getFullYear();
2424+
term = `${m < 10 ? '0' : ''}${m}/${d < 10 ? '0' : ''}${d}/${y}`;
2425+
}
2426+
2427+
let str = term.replace(/[\s"']+/g, ' ').trim();
2428+
if (str.indexOf(' ') >= 0) {
2429+
str = `'${str}'`;
2430+
}
2431+
2432+
return str;
2433+
};
2434+
2435+
for (let key of ['from', 'to', 'cc', 'bcc', 'subject', 'body']) {
2436+
if (search[key]) {
2437+
searchParts.push(`${key}:${escapeString(search[key])}`);
2438+
}
2439+
}
2440+
2441+
if (search.before || search.sentBefore) {
2442+
searchParts.push(`received<=${escapeString(search.before || search.sentBefore)}`);
2443+
}
2444+
2445+
if (search.since || search.sentSince) {
2446+
searchParts.push(`sent>=${escapeString(search.before || search.sentBefore)}`);
2447+
}
2448+
2449+
if (search.smaller) {
2450+
searchParts.push(`size<${Number(search.smaller) || 0}`);
2451+
}
2452+
2453+
if (search.larger) {
2454+
searchParts.push(`size>${Number(search.larger) || 0}`);
2455+
}
2456+
2457+
return searchParts.join(' ');
2458+
}
2459+
2460+
// convert IMAP SEARCH query object to a $filter query
2461+
prepareFilterQuery(search) {
23662462
search = search || {};
23672463

23682464
const filterParts = [];
@@ -2460,9 +2556,7 @@ class OutlookClient extends BaseClient {
24602556
}
24612557
}
24622558

2463-
return {
2464-
$filter: filterParts.join(' and ').trim()
2465-
};
2559+
return filterParts.join(' and ').trim();
24662560
}
24672561

24682562
async rollingBucketLock(key, value = '1') {
@@ -2537,9 +2631,9 @@ class OutlookClient extends BaseClient {
25372631
}
25382632

25392633
try {
2540-
let { page: cursorPage } = JSON.parse(Buffer.from(cursorStr, 'base64url'));
2634+
let { page: cursorPage, skipToken } = JSON.parse(Buffer.from(cursorStr, 'base64url'));
25412635
if (typeof cursorPage === 'number' && cursorPage >= 0) {
2542-
return cursorPage;
2636+
return { cursorPage, skipToken };
25432637
}
25442638
} catch (err) {
25452639
this.logger.error({ msg: 'Cursor parsing error', cursorStr, err });
@@ -2554,13 +2648,17 @@ class OutlookClient extends BaseClient {
25542648
return null;
25552649
}
25562650

2557-
encodeCursorString(cursorPage) {
2558-
if (typeof cursorPage !== 'number' || cursorPage < 0) {
2651+
encodeCursorString(cursorPage, skipToken) {
2652+
if ((typeof cursorPage !== 'number' && !skipToken) || cursorPage < 0) {
25592653
return null;
25602654
}
2655+
25612656
cursorPage = cursorPage || 0;
2657+
25622658
let type = 'ms';
2563-
return `${type}_${Buffer.from(JSON.stringify({ page: cursorPage })).toString('base64url')}`;
2659+
let encodedToken = `${type}_${Buffer.from(JSON.stringify({ page: cursorPage, skipToken })).toString('base64url')}`;
2660+
2661+
return encodedToken;
25642662
}
25652663
}
25662664

‎lib/oauth/gmail.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ class GmailOauth {
442442
} else if (payload && method === 'get') {
443443
let parsedUrl = new URL(url);
444444
for (let key of Object.keys(payload)) {
445-
if (typeof payload[key] === 'undefined') {
445+
if (typeof payload[key] === 'undefined' || payload[key] === null) {
446446
continue;
447447
}
448448
parsedUrl.searchParams.append(key, payload[key].toString());

‎lib/oauth/outlook.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -436,7 +436,7 @@ class OutlookOauth {
436436
} else if (payload && method === 'get') {
437437
let parsedUrl = new URL(url);
438438
for (let key of Object.keys(payload)) {
439-
if (typeof payload[key] === 'undefined') {
439+
if (typeof payload[key] === 'undefined' || payload[key] === null) {
440440
continue;
441441
}
442442
parsedUrl.searchParams.append(key, payload[key].toString());

0 commit comments

Comments
 (0)