Skip to content
This repository was archived by the owner on Apr 26, 2024. It is now read-only.

Commit 9f6ff6a

Browse files
H-ShayH-Shay
authored
Add not null constraint to column full_user_id of tables profiles and user_filters (#15537)
1 parent 77cda34 commit 9f6ff6a

File tree

10 files changed

+425
-4
lines changed

10 files changed

+425
-4
lines changed

‎changelog.d/15537.misc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add not null constraint to column full_user_id of tables profiles and user_filters.

‎synapse/storage/databases/main/filtering.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
LoggingDatabaseConnection,
2626
LoggingTransaction,
2727
)
28+
from synapse.storage.engines import PostgresEngine
2829
from synapse.types import JsonDict, UserID
2930
from synapse.util.caches.descriptors import cached
3031

@@ -40,6 +41,8 @@ def __init__(
4041
hs: "HomeServer",
4142
):
4243
super().__init__(database, db_conn, hs)
44+
self.server_name: str = hs.hostname
45+
self.database_engine = database.engine
4346
self.db_pool.updates.register_background_index_update(
4447
"full_users_filters_unique_idx",
4548
index_name="full_users_unique_idx",
@@ -48,6 +51,98 @@ def __init__(
4851
unique=True,
4952
)
5053

54+
self.db_pool.updates.register_background_update_handler(
55+
"populate_full_user_id_user_filters",
56+
self.populate_full_user_id_user_filters,
57+
)
58+
59+
async def populate_full_user_id_user_filters(
60+
self, progress: JsonDict, batch_size: int
61+
) -> int:
62+
"""
63+
Background update to populate the column `full_user_id` of the table
64+
user_filters from entries in the column `user_local_part` of the same table
65+
"""
66+
67+
lower_bound_id = progress.get("lower_bound_id", "")
68+
69+
def _get_last_id(txn: LoggingTransaction) -> Optional[str]:
70+
sql = """
71+
SELECT user_id FROM user_filters
72+
WHERE user_id > ?
73+
ORDER BY user_id
74+
LIMIT 1 OFFSET 50
75+
"""
76+
txn.execute(sql, (lower_bound_id,))
77+
res = txn.fetchone()
78+
if res:
79+
upper_bound_id = res[0]
80+
return upper_bound_id
81+
else:
82+
return None
83+
84+
def _process_batch(
85+
txn: LoggingTransaction, lower_bound_id: str, upper_bound_id: str
86+
) -> None:
87+
sql = """
88+
UPDATE user_filters
89+
SET full_user_id = '@' || user_id || ?
90+
WHERE ? < user_id AND user_id <= ? AND full_user_id IS NULL
91+
"""
92+
txn.execute(sql, (f":{self.server_name}", lower_bound_id, upper_bound_id))
93+
94+
def _final_batch(txn: LoggingTransaction, lower_bound_id: str) -> None:
95+
sql = """
96+
UPDATE user_filters
97+
SET full_user_id = '@' || user_id || ?
98+
WHERE ? < user_id AND full_user_id IS NULL
99+
"""
100+
txn.execute(
101+
sql,
102+
(
103+
f":{self.server_name}",
104+
lower_bound_id,
105+
),
106+
)
107+
108+
if isinstance(self.database_engine, PostgresEngine):
109+
sql = """
110+
ALTER TABLE user_filters VALIDATE CONSTRAINT full_user_id_not_null
111+
"""
112+
txn.execute(sql)
113+
114+
upper_bound_id = await self.db_pool.runInteraction(
115+
"populate_full_user_id_user_filters", _get_last_id
116+
)
117+
118+
if upper_bound_id is None:
119+
await self.db_pool.runInteraction(
120+
"populate_full_user_id_user_filters", _final_batch, lower_bound_id
121+
)
122+
123+
await self.db_pool.updates._end_background_update(
124+
"populate_full_user_id_user_filters"
125+
)
126+
return 1
127+
128+
await self.db_pool.runInteraction(
129+
"populate_full_user_id_user_filters",
130+
_process_batch,
131+
lower_bound_id,
132+
upper_bound_id,
133+
)
134+
135+
progress["lower_bound_id"] = upper_bound_id
136+
137+
await self.db_pool.runInteraction(
138+
"populate_full_user_id_user_filters",
139+
self.db_pool.updates._background_update_progress_txn,
140+
"populate_full_user_id_user_filters",
141+
progress,
142+
)
143+
144+
return 50
145+
51146
@cached(num_args=2)
52147
async def get_user_filter(
53148
self, user_localpart: str, filter_id: Union[int, str]

‎synapse/storage/databases/main/profile.py‎

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,14 @@
1515

1616
from synapse.api.errors import StoreError
1717
from synapse.storage._base import SQLBaseStore
18-
from synapse.storage.database import DatabasePool, LoggingDatabaseConnection
18+
from synapse.storage.database import (
19+
DatabasePool,
20+
LoggingDatabaseConnection,
21+
LoggingTransaction,
22+
)
1923
from synapse.storage.databases.main.roommember import ProfileInfo
20-
from synapse.types import UserID
24+
from synapse.storage.engines import PostgresEngine
25+
from synapse.types import JsonDict, UserID
2126

2227
if TYPE_CHECKING:
2328
from synapse.server import HomeServer
@@ -31,6 +36,8 @@ def __init__(
3136
hs: "HomeServer",
3237
):
3338
super().__init__(database, db_conn, hs)
39+
self.server_name: str = hs.hostname
40+
self.database_engine = database.engine
3441
self.db_pool.updates.register_background_index_update(
3542
"profiles_full_user_id_key_idx",
3643
index_name="profiles_full_user_id_key",
@@ -39,6 +46,97 @@ def __init__(
3946
unique=True,
4047
)
4148

49+
self.db_pool.updates.register_background_update_handler(
50+
"populate_full_user_id_profiles", self.populate_full_user_id_profiles
51+
)
52+
53+
async def populate_full_user_id_profiles(
54+
self, progress: JsonDict, batch_size: int
55+
) -> int:
56+
"""
57+
Background update to populate the column `full_user_id` of the table
58+
profiles from entries in the column `user_local_part` of the same table
59+
"""
60+
61+
lower_bound_id = progress.get("lower_bound_id", "")
62+
63+
def _get_last_id(txn: LoggingTransaction) -> Optional[str]:
64+
sql = """
65+
SELECT user_id FROM profiles
66+
WHERE user_id > ?
67+
ORDER BY user_id
68+
LIMIT 1 OFFSET 50
69+
"""
70+
txn.execute(sql, (lower_bound_id,))
71+
res = txn.fetchone()
72+
if res:
73+
upper_bound_id = res[0]
74+
return upper_bound_id
75+
else:
76+
return None
77+
78+
def _process_batch(
79+
txn: LoggingTransaction, lower_bound_id: str, upper_bound_id: str
80+
) -> None:
81+
sql = """
82+
UPDATE profiles
83+
SET full_user_id = '@' || user_id || ?
84+
WHERE ? < user_id AND user_id <= ? AND full_user_id IS NULL
85+
"""
86+
txn.execute(sql, (f":{self.server_name}", lower_bound_id, upper_bound_id))
87+
88+
def _final_batch(txn: LoggingTransaction, lower_bound_id: str) -> None:
89+
sql = """
90+
UPDATE profiles
91+
SET full_user_id = '@' || user_id || ?
92+
WHERE ? < user_id AND full_user_id IS NULL
93+
"""
94+
txn.execute(
95+
sql,
96+
(
97+
f":{self.server_name}",
98+
lower_bound_id,
99+
),
100+
)
101+
102+
if isinstance(self.database_engine, PostgresEngine):
103+
sql = """
104+
ALTER TABLE profiles VALIDATE CONSTRAINT full_user_id_not_null
105+
"""
106+
txn.execute(sql)
107+
108+
upper_bound_id = await self.db_pool.runInteraction(
109+
"populate_full_user_id_profiles", _get_last_id
110+
)
111+
112+
if upper_bound_id is None:
113+
await self.db_pool.runInteraction(
114+
"populate_full_user_id_profiles", _final_batch, lower_bound_id
115+
)
116+
117+
await self.db_pool.updates._end_background_update(
118+
"populate_full_user_id_profiles"
119+
)
120+
return 1
121+
122+
await self.db_pool.runInteraction(
123+
"populate_full_user_id_profiles",
124+
_process_batch,
125+
lower_bound_id,
126+
upper_bound_id,
127+
)
128+
129+
progress["lower_bound_id"] = upper_bound_id
130+
131+
await self.db_pool.runInteraction(
132+
"populate_full_user_id_profiles",
133+
self.db_pool.updates._background_update_progress_txn,
134+
"populate_full_user_id_profiles",
135+
progress,
136+
)
137+
138+
return 50
139+
42140
async def get_profileinfo(self, user_localpart: str) -> ProfileInfo:
43141
try:
44142
profile = await self.db_pool.simple_select_one(

‎synapse/storage/schema/__init__.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
SCHEMA_VERSION = 76 # remember to update the list below when updating
15+
SCHEMA_VERSION = 77 # remember to update the list below when updating
1616
"""Represents the expectations made by the codebase about the database schema
1717
1818
This should be incremented whenever the codebase changes its requirements on the
@@ -100,13 +100,19 @@
100100
101101
Changes in SCHEMA_VERSION = 76:
102102
- Adds a full_user_id column to tables profiles and user_filters.
103+
104+
Changes in SCHEMA_VERSION = 77
105+
- (Postgres) Add NOT VALID CHECK (full_user_id IS NOT NULL) to tables profiles and user_filters
103106
"""
104107

105108

106109
SCHEMA_COMPAT_VERSION = (
107110
# Queries against `event_stream_ordering` columns in membership tables must
108111
# be disambiguated.
109-
74
112+
#
113+
# insertions to the column `full_user_id` of tables profiles and user_filters can no
114+
# longer be null
115+
76
110116
)
111117
"""Limit on how far the synapse codebase can be rolled back without breaking db compat
112118

‎synapse/storage/schema/main/delta/77/01_add_profiles_not_valid_check.sql.postgres‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/* Copyright 2023 The Matrix.org Foundation C.I.C
2+
*
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
ALTER TABLE profiles ADD CONSTRAINT full_user_id_not_null CHECK (full_user_id IS NOT NULL) NOT VALID;

‎synapse/storage/schema/main/delta/77/02_add_user_filters_not_valid_check.sql.postgres‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/* Copyright 2023 The Matrix.org Foundation C.I.C
2+
*
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
ALTER TABLE user_filters ADD CONSTRAINT full_user_id_not_null CHECK (full_user_id IS NOT NULL) NOT VALID;

‎synapse/storage/schema/main/delta/77/03bg_populate_full_user_id_profiles.sql‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/* Copyright 2023 The Matrix.org Foundation C.I.C
2+
*
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES (7703, 'populate_full_user_id_profiles', '{}');

‎synapse/storage/schema/main/delta/77/04bg_populate_full_user_id_user_filters.sql‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/* Copyright 2023 The Matrix.org Foundation C.I.C
2+
*
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES (7704, 'populate_full_user_id_user_filters', '{}');

0 commit comments

Comments
 (0)