From 27b4395db04fdb879c66cdac599335ed7f4c45eb Mon Sep 17 00:00:00 2001 From: Tatu Wikman Date: Fri, 7 Apr 2023 13:56:03 +0300 Subject: [PATCH 1/8] Docs: Add Nginx loadbalancing example with sticky mxid for workers Add example nginx configuration snippet that * does load balancing for workers * respects mxid part of the token * from both url parameter and auth header * and handles since parameter Thanks to @olmari for pushing me to write this and testing the configs Signed-off-by: Tatu Wikman --- docs/workers.md | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/docs/workers.md b/docs/workers.md index e9a477d32c89..ffc4844c14aa 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -333,6 +333,76 @@ when a user logs in on a new device and can be *very* resource intensive, so isolating these requests will stop them from interfering with other users ongoing syncs. +Example `nginx` configuration snippet that handles the cases above. This is just an +example and probably requires some changes according to your particular setup: + +```js +# Choose sync worker based on the existence of "since" query parameter +map $arg_since $sync { + default synapse_sync; + '' synapse_initial_sync; +} + +# Extract username from access token passed as URL parameter +map $arg_access_token $accesstoken_from_urlparam { + # Defaults to just passing back the whole accesstoken + default $arg_access_token; + # Try to extract username part from accesstoken URL parameter + "~syt_(?.*?)_.*" $username; +} + +# Extract username from access token passed as authorization header +map $http_authorization $mxid_localpart { + # Defaults to just passing back the whole accesstoken + default $http_authorization; + # Try to extract username part from accesstoken header + "~Bearer syt_(?.*?)_.*" $username; + # if no authorization-header exist, try mapper for URL parameter "access_token" + "" $accesstoken_from_urlparam; +} + +upstream synapse_initial_sync { + # Use the username mapper result for hash key + hash $mxid_localpart consistent; + server 127.0.0.1:8016; + server 127.0.0.1:8036; +} + +upstream synapse_sync { + # Use the username mapper result for hash key + hash $mxid_localpart consistent; + server 127.0.0.1:8013; + server 127.0.0.1:8037; + server 127.0.0.1:8038; + server 127.0.0.1:8039; +} + +# Sync initial/normal +location ~ ^/_matrix/client/(r0|v3)/sync$ { + include snippets/matrix-proxy-headers.conf; + proxy_pass http://$sync; + proxy_read_timeout 1h; +} + +# Normal sync +location ~ ^/_matrix/client/(api/v1|r0|v3)/events$ { + include snippets/matrix-proxy-headers.conf; + proxy_pass http://synapse_sync; +} + +# Initial_sync +location ~ ^/_matrix/client/(api/v1|r0|v3)/initialSync$ { + include snippets/matrix-proxy-headers.conf; + proxy_pass http://synapse_initial_sync; + proxy_read_timeout 1h; +} +location ~ ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ { + include snippets/matrix-proxy-headers.conf; + proxy_pass http://synapse_initial_sync; + proxy_read_timeout 1h; +} +``` + Federation and client requests can be balanced via simple round robin. The inbound federation transaction request `^/_matrix/federation/v1/send/` From 71ea1120e4adb59ee64714ffabaa8f59e9663d84 Mon Sep 17 00:00:00 2001 From: Tatu Wikman Date: Fri, 7 Apr 2023 14:04:22 +0300 Subject: [PATCH 2/8] Add changelog entry Signed-off-by: Tatu Wikman --- changelog.d/15411.docs | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/15411.docs diff --git a/changelog.d/15411.docs b/changelog.d/15411.docs new file mode 100644 index 000000000000..c23a8df04a71 --- /dev/null +++ b/changelog.d/15411.docs @@ -0,0 +1 @@ +Docs: Add Nginx loadbalancing example with sticky mxid for workers. From c7d54aa71cf0e7893ebf4d3248f2b49d6248412d Mon Sep 17 00:00:00 2001 From: Tatu Wikman Date: Fri, 7 Apr 2023 20:15:21 +0300 Subject: [PATCH 3/8] Update codeblock formatter Co-authored-by: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> --- docs/workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/workers.md b/docs/workers.md index ffc4844c14aa..876bab78333c 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -336,7 +336,7 @@ syncs. Example `nginx` configuration snippet that handles the cases above. This is just an example and probably requires some changes according to your particular setup: -```js +```nginx # Choose sync worker based on the existence of "since" query parameter map $arg_since $sync { default synapse_sync; From 338de72b0a60aa73af653cfbfa7ab27c681efce5 Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Fri, 7 Apr 2023 19:49:23 +0300 Subject: [PATCH 4/8] Remove indirectly related nginx-config Signed-off-by: Sami Olmari --- docs/workers.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/workers.md b/docs/workers.md index 876bab78333c..104c56aa2374 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -379,27 +379,20 @@ upstream synapse_sync { # Sync initial/normal location ~ ^/_matrix/client/(r0|v3)/sync$ { - include snippets/matrix-proxy-headers.conf; proxy_pass http://$sync; - proxy_read_timeout 1h; } # Normal sync location ~ ^/_matrix/client/(api/v1|r0|v3)/events$ { - include snippets/matrix-proxy-headers.conf; proxy_pass http://synapse_sync; } # Initial_sync location ~ ^/_matrix/client/(api/v1|r0|v3)/initialSync$ { - include snippets/matrix-proxy-headers.conf; proxy_pass http://synapse_initial_sync; - proxy_read_timeout 1h; } location ~ ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ { - include snippets/matrix-proxy-headers.conf; proxy_pass http://synapse_initial_sync; - proxy_read_timeout 1h; } ``` From 745d820281c776ded46a9c68e86ad8779a64c81b Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Thu, 13 Apr 2023 21:54:46 +0300 Subject: [PATCH 5/8] Proper definition of action how to target username for worker Signed-off-by: Sami Olmari --- docs/workers.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/workers.md b/docs/workers.md index 104c56aa2374..7193248907ef 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -323,8 +323,7 @@ load balancing can be done in different ways. For `/sync` and `/initialSync` requests it will be more efficient if all requests from a particular user are routed to a single instance. This can -be done e.g. in nginx via IP `hash $http_x_forwarded_for;` or via -`hash $http_authorization consistent;` which contains the users access token. +be done e.g. in nginx via extracting username from the users access token. Admins may additionally wish to separate out `/sync` requests that have a `since` query parameter from those that don't (and From 8ec83ec7d2565b9c35710bb582afb59968a69c54 Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Thu, 13 Apr 2023 22:02:09 +0300 Subject: [PATCH 6/8] Change "nginx" to general "reverse proxy" as it's concept now. Signed-off-by: Sami Olmari --- docs/workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/workers.md b/docs/workers.md index 7193248907ef..c05b8a15230f 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -323,7 +323,7 @@ load balancing can be done in different ways. For `/sync` and `/initialSync` requests it will be more efficient if all requests from a particular user are routed to a single instance. This can -be done e.g. in nginx via extracting username from the users access token. +be done reverse proxy via extracting username from the users access token. Admins may additionally wish to separate out `/sync` requests that have a `since` query parameter from those that don't (and From 43837730907ee63120c20037abac31cd2f7c93d2 Mon Sep 17 00:00:00 2001 From: Sami Olmari Date: Thu, 13 Apr 2023 22:19:22 +0300 Subject: [PATCH 7/8] Wording in better English Co-authored-by: Tatu Wikman --- docs/workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/workers.md b/docs/workers.md index c05b8a15230f..3625e63bf017 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -323,7 +323,7 @@ load balancing can be done in different ways. For `/sync` and `/initialSync` requests it will be more efficient if all requests from a particular user are routed to a single instance. This can -be done reverse proxy via extracting username from the users access token. +be done in reverse proxy by extracting username part from the users access token. Admins may additionally wish to separate out `/sync` requests that have a `since` query parameter from those that don't (and From 19b4597b8cd2aa7168f6e24117326617963b2c8f Mon Sep 17 00:00:00 2001 From: Tatu Wikman Date: Wed, 26 Apr 2023 08:41:36 +0300 Subject: [PATCH 8/8] rename changelog entry to have correct extension --- changelog.d/{15411.docs => 15411.doc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{15411.docs => 15411.doc} (100%) diff --git a/changelog.d/15411.docs b/changelog.d/15411.doc similarity index 100% rename from changelog.d/15411.docs rename to changelog.d/15411.doc