Skip to content

Some more FORCE INDEX #870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
squeaky-pl merged 5 commits into from
Sep 3, 2024
Merged

Some more FORCE INDEX #870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cd8de37
One more FORCE INDEX
squeaky-pl Aug 29, 2024
9ec42cd
Even more
squeaky-pl Aug 29, 2024
155682a
Even more
squeaky-pl Aug 29, 2024
fb77cf6
Even more
squeaky-pl Aug 29, 2024
a81b436
Oops
squeaky-pl Aug 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion inbox/mailsync/backends/gmail.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,14 @@ def resync_uids_impl(self):
.options(
load_only("msg_uid"), joinedload("message").load_only("g_msgid")
)
.filter_by(account_id=self.account_id, folder_id=self.folder_id)
.filter(

@squeaky-pl squeaky-pl Aug 29, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Covering index since we filter by account_id, folder_id and retrieve msg_uid.

ImapUid.account_id == self.account_id,
ImapUid.folder_id == self.folder_id,
)
.with_hint(
ImapUid,
"FORCE INDEX (ix_imapuid_account_id_folder_id_msg_uid_desc)",
)
)

chunk_size = 1000
Expand Down
18 changes: 14 additions & 4 deletions inbox/mailsync/backends/imap/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,16 @@ def update_metadata(account_id, folder_id, folder_role, new_flags, session):

account = Account.get(account_id, session)
change_count = 0
for item in session.query(ImapUid).filter(
ImapUid.account_id == account_id,
ImapUid.msg_uid.in_(new_flags),
ImapUid.folder_id == folder_id,
for item in (
session.query(ImapUid)
.filter(
ImapUid.account_id == account_id,
ImapUid.folder_id == folder_id,
ImapUid.msg_uid.in_(new_flags),
)
.with_hint(
ImapUid, "FORCE INDEX (ix_imapuid_account_id_folder_id_msg_uid_desc)"
)
Comment on lines +160 to +168

@squeaky-pl squeaky-pl Aug 29, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a covering index since we retrieve other columns, but does not make anything slower, just switches the index.

):
flags = new_flags[item.msg_uid].flags
labels = getattr(new_flags[item.msg_uid], "labels", None)
Expand Down Expand Up @@ -204,6 +210,10 @@ def remove_deleted_uids(account_id, folder_id, uids):
ImapUid.folder_id == folder_id,
ImapUid.msg_uid == uid,
)
.with_hint(
ImapUid,
"FORCE INDEX (ix_imapuid_account_id_folder_id_msg_uid_desc)",
)
Comment on lines +213 to +216

@squeaky-pl squeaky-pl Aug 29, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a covering index since we retrieve other columns, but does not make anything slower, just switches the index.

.first()
)
if imapuid is None:
Expand Down
25 changes: 17 additions & 8 deletions inbox/mailsync/backends/imap/generic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -517,8 +517,13 @@ def resync_uids_impl(self):
with session_scope(self.namespace_id) as db_session:
invalid_uids = {
uid
for uid, in db_session.query(ImapUid.msg_uid).filter_by(
account_id=self.account_id, folder_id=self.folder_id
for uid, in db_session.query(ImapUid.msg_uid)
.filter(
ImapUid.account_id == self.account_id,
ImapUid.folder_id == self.folder_id,
)
.with_hint(
ImapUid, "FORCE INDEX(ix_imapuid_account_id_folder_id_msg_uid_desc)"
Comment on lines +520 to +526

@squeaky-pl squeaky-pl Aug 29, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Covering index since we filter by account_id, folder_id and retrieve msg_uid.

)
}
with self.syncmanager_lock:
Expand All @@ -545,20 +550,24 @@ def create_message(

# Check if we somehow already saved the imapuid (shouldn't happen, but
# possible due to race condition). If so, don't commit changes.
existing_imapuid = (
imapuid_exists = db_session.query(
db_session.query(ImapUid)
.filter(
ImapUid.account_id == account.id,
ImapUid.folder_id == folder.id,
ImapUid.msg_uid == raw_message.uid,
)
.first()
)
if existing_imapuid is not None:
.with_hint(
ImapUid, "FORCE INDEX(ix_imapuid_account_id_folder_id_msg_uid_desc)"
)
.exists()
).scalar()
if imapuid_exists:
log.warning(
"Expected to create imapuid, but existing row found",
remote_msg_uid=raw_message.uid,
existing_imapuid=existing_imapuid.id,

@squeaky-pl squeaky-pl Aug 29, 2024 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not really interesting to know the id column here, all we care about is the triple (account_id, folder_id, imap_uid) is unique, which can be known from the unique index (account_id, folder_id, imap_uid DESC) which is covering for this query. Faster.

account_id=account.id,
folder_id=folder.id,
msg_uid=raw_message.uid,
)
return None

Expand Down