Synapse workers #456
+1,785
−233
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -101,6 +101,114 @@ | |
| } | ||
| {% endif %} | ||
|
|
||
| {% if synchrotron_workers %} | ||
| {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #} | ||
| location /_matrix/client/r0/sync { | ||
| proxy_pass http://synchrotron$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| location /_matrix/client/r0/events { | ||
| proxy_pass http://synchrotron$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| location /_matrix/client/r0/initialSync { | ||
| proxy_pass http://synchrotron$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ { | ||
| proxy_pass http://synchrotron$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {% endif %} | ||
|
|
||
| {% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %} | ||
| {% if client_reader_worker %} | ||
| {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} | ||
| location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { | ||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
| resolver 127.0.0.11 valid=5s; | ||
| set $backend "matrix-synapse:{{ client_reader_worker.port }}"; | ||
| proxy_pass http://$backend$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {% endif %} | ||
|
|
||
| {% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %} | ||
| {% if media_repository_worker %} | ||
| {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} | ||
| location /_matrix/media/ { | ||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
| resolver 127.0.0.11 valid=5s; | ||
| set $backend "matrix-synapse:{{ media_repository_worker.port }}"; | ||
| proxy_pass http://$backend$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} | ||
| location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { | ||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
| resolver 127.0.0.11 valid=5s; | ||
| set $backend "matrix-synapse:{{ media_repository_worker.port }}"; | ||
| proxy_pass http://$backend$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {% endif %} | ||
|
|
||
| {% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %} | ||
| {% if event_creator_worker %} | ||
| {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} | ||
| location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { | ||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
| resolver 127.0.0.11 valid=5s; | ||
| set $backend "matrix-synapse:{{ event_creator_worker.port }}"; | ||
| proxy_pass http://$backend$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {% endif %} | ||
|
|
||
| {% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %} | ||
| {% if frontend_proxy_worker %} | ||
| {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} | ||
| location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { | ||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
| resolver 127.0.0.11 valid=5s; | ||
| set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; | ||
| proxy_pass http://$backend$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {% if not matrix_synapse_use_presence %} | ||
| location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { | ||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
| resolver 127.0.0.11 valid=5s; | ||
| set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; | ||
| proxy_pass http://$backend$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {% endif %} | ||
| {% endif %} | ||
|
|
||
| {% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %} | ||
| {% if user_dir_worker %} | ||
| {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} | ||
| location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { | ||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
| resolver 127.0.0.11 valid=5s; | ||
| set $backend "matrix-synapse:{{ user_dir_worker.port }}"; | ||
| proxy_pass http://$backend$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {% endif %} | ||
|
|
||
| {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} | ||
| {{- configuration_block }} | ||
| {% endfor %} | ||
|
|
@@ -174,6 +282,19 @@ | |
| } | ||
| {% endmacro %} | ||
|
|
||
| {% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %} | ||
| {% if synchrotron_workers %} | ||
| upstream synchrotron { | ||
| # ensures that requests from the same client will always be passed | ||
| # to the same server (except when this server is unavailable) | ||
| ip_hash; | ||
|
|
||
| {% for synchrotron_worker in synchrotron_workers %} | ||
| server "matrix-synapse:{{ synchrotron_worker.port }}"; | ||
| {% endfor %} | ||
| } | ||
| {% endif %} | ||
|
|
||
| server { | ||
| listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; | ||
| server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; | ||
|
|
@@ -255,6 +376,19 @@ server { | |
| ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; | ||
| {% endif %} | ||
|
|
||
| {% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} | ||
| {% if federation_reader_worker %} | ||
| {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} | ||
| location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { | ||
|
There was a problem hiding this comment. /_matrix/federation/v1/groups/ should only be allowed GET requests; this configuration will cause federated community invites to fail. reference here: https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfederation_reader There was a problem hiding this comment. Thanks for the feedback.. Is this theoretical or have you experienced it on an actual system? There was a problem hiding this comment. Yes, I experienced it myself when trying to set up a fresh homeserver so I just removed the groups endpoint so the main synapse thread is handling it instead for now. |
||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
| resolver 127.0.0.11 valid=5s; | ||
| set $backend "matrix-synapse:{{ federation_reader_worker.port }}"; | ||
| proxy_pass http://$backend$request_uri; | ||
| proxy_set_header Host $host; | ||
| proxy_set_header X-Forwarded-For $remote_addr; | ||
| } | ||
| {% endif %} | ||
|
|
||
| location / { | ||
| {% if matrix_nginx_proxy_enabled %} | ||
| {# Use the embedded DNS resolver in Docker containers to discover the service #} | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| --- | ||
| # a negative when condition will not actually prevent ansible from executing loops in imported tasks! | ||
|
|
||
| - import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" | ||
| when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" | ||
|
|
||
| - import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml" | ||
| when: "not matrix_synapse_workers_enabled|bool" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| --- | ||
|
|
||
| - name: Ensure synapse worker base service file installed | ||
| template: | ||
| src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2" | ||
| dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" | ||
| mode: 0644 | ||
| register: matrix_synapse_worker_systemd_service_result | ||
|
|
||
| - name: Ensure previous worker service symlinks are cleaned (FIXME) | ||
| file: | ||
| path: "{{ item.root + '/' + item.path }}" | ||
| state: absent | ||
| when: | ||
| - matrix_synapse_workers_enabled|bool | ||
| - item.state == 'link' | ||
| - item.path is match('matrix-synapse-worker@.*\\.service') | ||
| with_filetree: | ||
| - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" | ||
|
|
||
| - name: Ensure systemd reloaded the worker service unit | ||
| service: | ||
| daemon_reload: yes | ||
|
|
||
| - name: Ensure individual worker service symlinks exist | ||
| service: | ||
| name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" | ||
| enabled: true | ||
| with_items: "{{ matrix_synapse_workers_enabled_list }}" | ||
|
|
||
| - name: Ensure creation of specific worker configs | ||
| template: | ||
| src: "{{ role_path }}/templates/synapse/worker.yaml.j2" | ||
| dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml" | ||
| with_list: "{{ matrix_synapse_workers_enabled_list }}" | ||
|
|
||
| - name: Add workers to synapse.wants list | ||
| set_fact: | ||
| matrix_synapse_systemd_wanted_services_list: > | ||
| {{ matrix_synapse_systemd_wanted_services_list + | ||
| ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }} | ||
| with_items: "{{ matrix_synapse_workers_enabled_list }}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| --- | ||
|
|
||
| - name: Populate service facts | ||
| service_facts: | ||
|
|
||
| - name: Ensure any worker services are stopped | ||
| service: | ||
| name: "{{ item.key }}" | ||
| state: stopped | ||
| with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}" | ||
|
|
||
| # As we cannot know the ports of workers removed from the enabled_list.. | ||
| # => .. just kill them all (FIXME?) | ||
| - name: Ensure previous worker service symlinks are cleaned | ||
| file: | ||
| path: "{{ item.root + '/' + item.path }}" | ||
| state: absent | ||
| when: | ||
| - not matrix_synapse_workers_enabled|bool | ||
| - item.state == 'link' | ||
| - item.path is match('matrix-synapse-worker@.*\\.service') | ||
| with_filetree: | ||
| - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" | ||
|
|
||
| - name: Ensure synapse worker base service file gets removed | ||
| file: | ||
| path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" | ||
| state: absent | ||
| register: matrix_synapse_worker_systemd_service_result | ||
|
|
||
| - name: Remove workers from synapse.wants list | ||
| set_fact: | ||
| matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}" | ||
| with_items: "{{ matrix_synapse_workers_avail_list }}" | ||
|
|
||
| - name: Ensure systemd noticed removal of worker service units | ||
| service: | ||
| daemon_reload: yes |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| #jinja2: lstrip_blocks: "True" | ||
| # c.f. https://github.com/matrix-org/synapse/pull/4662 | ||
| [Unit] | ||
| Description=Synapse Matrix Worker | ||
| After=matrix-synapse.service | ||
| BindsTo=matrix-synapse.service | ||
|
|
||
| [Service] | ||
| Type=simple | ||
|
|
||
| # Intentional delay, so that the homeserver (we likely depend on) can manage to start. | ||
| ExecStartPre=/bin/sleep 5 | ||
|
|
||
| # systemd ftw 🤦♂️ | ||
| # https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 | ||
| ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ | ||
| exec /usr/bin/docker exec \ | ||
| --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ | ||
| matrix-synapse \ | ||
| python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" | ||
|
|
||
| ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i | ||
spantaleev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| KillMode=process | ||
| Restart=always | ||
| RestartSec=10 | ||
| SyslogIdentifier=matrix-synapse-%i | ||
|
|
||
| [Install] | ||
| WantedBy=matrix-synapse.service | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Two of the endpoints here can only handle GET requests, and there are also two more not included here. The following list is GET-only for the client_reader worker:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The "location ^/..." here, for a regex match needs to be "location ~ ^..." doesn't it?