Synapse workers

.gitignore

@@ -3,5 +3,7 @@
 !/inventory/host_vars/.gitkeep
 !/inventory/scripts
 /roles/*/files/scratchpad
+/roles/matrix-synapse/files/workers.upstream-documentation.md
+/roles/matrix-synapse/vars/workers.yml
 .DS_Store
 .python-version

docs/configuring-playbook-synapse.md

@@ -18,6 +18,23 @@ Alternatively, **if there is no pre-defined variable** for a Synapse setting you
 - or, if extending the configuration is still not powerful enough for your needs, you can **override the configuration completely** using `matrix_synapse_configuration` (or `matrix_synapse_configuration_yaml`). You can find information about this in [`roles/matrix-synapse/defaults/main.yml`](../roles/matrix-synapse/defaults/main.yml).
 
 
+## Load balancing with workers
+To have synapse gracefully handle thousands of users, worker support should be enabled. It factors out some homeserver tasks and spreads the load of incoming client and server-to-server traffic between multiple processes. More information can be found at https://github.com/matrix-org/synapse/blob/master/docs/workers.md (which, coincidentally, also is the file which an awk script extracts the endpoint URLs from when running with tag `setup-synapse`).
+
+To enable synapse worker support, set
+
+```yaml
+matrix_synapse_workers_enabled: true
+```
+
+in your `inventory/host_vars/matrix.DOMAIN/vars.yml` file.
+There, you can also override the default `matrix_synapse_workers_enabled_list` from [`roles/matrix-synapse/defaults/main.yml`](../roles/matrix-synapse/defaults/main.yml).
+
+If you are not using the inbuilt nginx proxy container but an instance managed by yourself, you are currently on your own as the template needs yet to be adapted to better support this use case.
+
+In case any problems occur, make sure to have a look at the [list of synapse issues about workers](https://github.com/matrix-org/synapse/issues?q=workers+in%3Atitle) and your `journalctl --unit 'matrix-*'`.
+
+
 ## Synapse Admin
 
 Certain Synapse administration tasks (managing users and rooms, etc.) can be performed via a web user-interace, if you install [Synapse Admin](configuring-playbook-synapse-admin.md).

group_vars/matrix_servers

@@ -830,6 +830,15 @@ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_sans_container: "{{ m
 
 matrix_nginx_proxy_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}"
 
+matrix_nginx_proxy_synapse_presence_disabled: "{{ not matrix_synapse_use_presence }}"
+
+matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}"
+matrix_nginx_proxy_synapse_workers_list: "{{ matrix_synapse_workers_enabled_list }}"
+matrix_nginx_proxy_synapse_generic_worker_locations: "{{ matrix_synapse_workers_generic_worker_endpoints|default([]) }}"
+matrix_nginx_proxy_synapse_media_repository_locations: "{{ matrix_synapse_workers_media_repository_endpoints|default([]) }}"
+matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}"
+matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}"
+
 matrix_nginx_proxy_systemd_wanted_services_list: |
   {{
     (['matrix-synapse.service'])
@@ -896,6 +905,22 @@ matrix_postgres_db_name: "homeserver"
 
 
 
+######################################################################
+#
+# matrix-redis
+#
+######################################################################
+
+matrix_redis_enabled: "{{ matrix_synapse_workers_enabled }}"
+
+######################################################################
+#
+# /matrix-redis
+#
+######################################################################
+
+
+
 ######################################################################
 #
 # matrix-client-element
@@ -1035,6 +1060,11 @@ matrix_synapse_systemd_wanted_services_list: |
     (['matrix-mailer.service'] if matrix_mailer_enabled else [])
   }}
 
+# Synapse workers (used for parallel load-scaling) need Redis for IPC.
+matrix_synapse_redis_enabled: "{{ matrix_redis_enabled }}"
+matrix_synapse_redis_host: "{{ 'matrix-redis' if matrix_redis_enabled else '' }}"
+matrix_synapse_redis_password: "{{ matrix_redis_connection_password if matrix_redis_enabled else '' }}"
+
 ######################################################################
 #
 # /matrix-synapse

roles/matrix-nginx-proxy/defaults/main.yml

@@ -283,3 +283,12 @@ matrix_ssl_pre_obtaining_required_service_start_wait_time_seconds: 60
 # nginx status page configurations.
 matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false
 matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_default_ipv4.address }}']
+
+
+# synapse worker activation and endpoint mappings
+matrix_nginx_proxy_synapse_workers_enabled: false
+matrix_nginx_proxy_synapse_workers_list: []
+matrix_nginx_proxy_synapse_generic_worker_locations: []
+matrix_nginx_proxy_synapse_media_repository_locations: []
+matrix_nginx_proxy_synapse_user_dir_locations: []
+matrix_nginx_proxy_synapse_frontend_proxy_locations: []

roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2

@@ -68,6 +68,9 @@
 	{% endif %}
 
 	{% if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled %}
+	# NOTE: This redirects user lookup requests to the identity server instead of
+	# synapse, so user_dir_workers endpoints listed further down in this file will
+	# not be reached and workers of this kind should be disabled for consistency.
 	location ^~ /_matrix/client/r0/user_directory/search {
 		{% if matrix_nginx_proxy_enabled %}
 			{# Use the embedded DNS resolver in Docker containers to discover the service #}
@@ -101,6 +104,70 @@
 	}
 	{% endif %}
 
+	{% if matrix_nginx_proxy_synapse_workers_enabled %}
+		{# Workers redirects BEGIN #}
+
+		{% if generic_workers %}
+			# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker
+			{% for location in matrix_nginx_proxy_synapse_generic_worker_locations %}
+			location ~ {{ location }} {
+				proxy_pass http://generic_worker_upstream$request_uri;
+				proxy_set_header Host $host;
+				proxy_set_header X-Forwarded-For $remote_addr;
+			}
+			{% endfor %}
+			# FIXME: add GET ^/_matrix/federation/v1/groups/
+		{% endif %}
+
+		{% if media_repository_workers %}
+			# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository
+			{% for location in matrix_nginx_proxy_synapse_media_repository_locations %}
+			location ~ {{ location }} {
+				proxy_pass http://media_repository_upstream$request_uri;
+				proxy_set_header Host $host;
+				proxy_set_header X-Forwarded-For $remote_addr;
+
+				client_body_buffer_size 25M;
+				client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb }}M;
+				proxy_max_temp_file_size 0;
+			}
+			{% endfor %}
+		{% endif %}
+
+		{% if user_dir_workers %}
+			# FIXME: obsolete if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled is set
+			# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir
+			{% for location in matrix_nginx_proxy_synapse_user_dir_locations %}
+			location ~ {{ location }} {
+				proxy_pass http://user_dir_upstream$request_uri;
+				proxy_set_header Host $host;
+				proxy_set_header X-Forwarded-For $remote_addr;
+			}
+			{% endfor %}
+		{% endif %}
+
+		{% if frontend_proxy_workers %}
+			# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy
+			{% for location in matrix_nginx_proxy_synapse_frontend_proxy_locations %}
+			location ~ {{ location }} {
+				proxy_pass http://frontend_proxy_upstream$request_uri;
+				proxy_set_header Host $host;
+				proxy_set_header X-Forwarded-For $remote_addr;
+			}
+			{% endfor %}
+			{% if matrix_nginx_proxy_synapse_presence_disabled %}
+			# FIXME: keep in sync with synapse workers documentation manually
+			location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
+				proxy_pass http://frontend_proxy_upstream$request_uri;
+				proxy_set_header Host $host;
+				proxy_set_header X-Forwarded-For $remote_addr;
+			}
+			{% endif %}
+		{% endif %}
+		{# Workers redirects END #}
+	{% endif %}
+
+
 	{% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %}
 		{{- configuration_block }}
 	{% endfor %}
@@ -159,6 +226,50 @@
 	}
 {% endmacro %}
 
+{% set generic_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'generic_worker')|list %}
+{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'media_repository')|list %}
+{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'user_dir')|list %}
+{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_list|selectattr('type', 'equalto', 'frontend_proxy')|list %}
+{% if matrix_nginx_proxy_synapse_workers_enabled %}
+	# Round Robin "upstream" pools for workers
+
+	{% if generic_workers %}
+	upstream generic_worker_upstream {
+		# ensures that requests from the same client will always be passed
+		# to the same server (except when this server is unavailable)
+		ip_hash;
+
+		{% for worker in generic_workers %}
+		server "matrix-synapse:{{ worker.port }}";
+		{% endfor %}
+	}
+	{% endif %}
+
+	{% if frontend_proxy_workers %}
+	upstream frontend_proxy_upstream {
+		{% for worker in frontend_proxy_workers %}
+		server "matrix-synapse:{{ worker.port }}";
+		{% endfor %}
+	}
+	{% endif %}
+
+	{% if media_repository_workers %}
+	upstream media_repository_upstream {
+		{% for worker in media_repository_workers %}
+		server "matrix-synapse:{{ worker.port }}";
+		{% endfor %}
+	}
+	{% endif %}
+
+	{% if user_dir_workers %}
+	upstream user_dir_upstream {
+		{% for worker in user_dir_workers %}
+		server "matrix-synapse:{{ worker.port }}";
+		{% endfor %}
+	}
+	{% endif %}
+{% endif %}
+
 server {
 	listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }};
 	server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }};
@@ -240,6 +351,34 @@ server {
 		ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
 	{% endif %}
 
+	{% if matrix_nginx_proxy_synapse_workers_enabled %}
+		{% if generic_workers %}
+			# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker
+			{% for location in matrix_nginx_proxy_synapse_generic_worker_locations %}
+			location ~ {{ location }} {
+				proxy_pass http://generic_worker_upstream$request_uri;
+				proxy_set_header Host $host;
+				proxy_set_header X-Forwarded-For $remote_addr;
+			}
+			{% endfor %}
+			# FIXME: add GET ^/_matrix/federation/v1/groups/
+		{% endif %}
+		{% if media_repository_workers %}
+			# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository
+			{% for location in matrix_nginx_proxy_synapse_media_repository_locations %}
+			location ~ {{ location }} {
+				proxy_pass http://media_repository_upstream$request_uri;
+				proxy_set_header Host $host;
+				proxy_set_header X-Forwarded-For $remote_addr;
+
+				client_body_buffer_size 25M;
+				client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb }}M;
+				proxy_max_temp_file_size 0;
+			}
+			{% endfor %}
+		{% endif %}
+	{% endif %}
+
 	location / {
 		{% if matrix_nginx_proxy_enabled %}
 			{# Use the embedded DNS resolver in Docker containers to discover the service #}

roles/matrix-postgres/defaults/main.yml

@@ -23,6 +23,10 @@ matrix_postgres_docker_image_force_pull: "{{ matrix_postgres_docker_image_to_use
 # A list of extra arguments to pass to the container
 matrix_postgres_container_extra_arguments: []
 
+# A list of extra arguments to pass to the postgres process
+# e.g. "-c 'max_connections=200'"
+matrix_postgres_process_extra_arguments: []
+
 # Controls whether the matrix-postgres container exposes a port (tcp/5432 in the
 # container) that can be used to access the database from outside the container (e.g. with psql)
 #

roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2

@@ -26,7 +26,11 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-postgres \
 			{% for arg in matrix_postgres_container_extra_arguments %}
 			{{ arg }} \
 			{% endfor %}
-			{{ matrix_postgres_docker_image_to_use }}
+			{{ matrix_postgres_docker_image_to_use }} \
+			postgres \
+			{% for arg in matrix_postgres_process_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
 
 ExecStop=-{{ matrix_host_command_docker }} stop matrix-postgres
 ExecStop=-{{ matrix_host_command_docker }} rm matrix-postgres

roles/matrix-redis/defaults/main.yml

@@ -0,0 +1,22 @@
+matrix_redis_enabled: true
+
+matrix_redis_connection_password: ""
+
+matrix_redis_base_path: "{{ matrix_base_data_path }}/redis"
+matrix_redis_data_path: "{{ matrix_redis_base_path }}/data"
+
+matrix_redis_docker_image_v5: "redis:5.0-alpine"
+matrix_redis_docker_image_v6: "redis:6.0-alpine"
+matrix_redis_docker_image_latest: "{{ matrix_redis_docker_image_v6 }}"
+matrix_redis_docker_image_to_use: '{{ matrix_redis_docker_image_latest }}'
+
+matrix_redis_docker_image_force_pull: "{{ matrix_redis_docker_image_to_use.endswith(':latest') }}"
+
+# A list of extra arguments to pass to the container
+matrix_redis_container_extra_arguments: []
+
+# Controls whether the matrix-redis container exposes a port (tcp/6379 in the container)
+# that can be used to access redis from outside the container
+#
+# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:6379"), or empty string to not expose.
+matrix_redis_container_redis_bind_port: ""

roles/matrix-redis/tasks/init.yml

@@ -0,0 +1,3 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-redis'] }}"
+  when: matrix_redis_enabled|bool

roles/matrix-redis/tasks/main.yml

@@ -0,0 +1,9 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/setup_redis.yml"
+  when: run_setup|bool
+  tags:
+    - setup-all
+    - setup-redis