From cdad12010466ecddb493bdebf248e531e5746087 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 5 Apr 2022 19:11:28 +0100 Subject: [PATCH 01/12] Poetry: use locked environment in Docker images Pulled out of #12337. --- .dockerignore | 4 +-- docker/Dockerfile | 63 ++++++++++++++++++++++++++++++----------------- docker/start.py | 12 ++++----- 3 files changed, 48 insertions(+), 31 deletions(-) diff --git a/.dockerignore b/.dockerignore index 434231fce9fc..7809863ef328 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,8 +4,8 @@ # things to include !docker !synapse -!MANIFEST.in !README.rst -!setup.py +!pyproject.toml +!poetry.lock **/__pycache__ diff --git a/docker/Dockerfile b/docker/Dockerfile index 24b5515eb99e..fc243053789d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,18 +16,31 @@ ARG PYTHON_VERSION=3.9 +FROM docker.io/python:${PYTHON_VERSION}-slim as base + ### ### Stage 0: builder ### -FROM docker.io/python:${PYTHON_VERSION}-slim as builder -# install the OS build deps -# +# Irritatingly, there is no blessed guide on how to distribute an application with its +# poetry-managed environment in a docker image. For a while, +# `poetry export | pip install -r /dev/stdin` seemed plausible but is limited by bugs +# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released. +# This is inspired from: +# https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865 +# https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc +FROM base as builder + # RUN --mount is specific to buildkit and is documented at # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. -# Here we use it to set up a cache for apt, to improve rebuild speeds on -# slow connections. -# +# Here we use it to set up a cache for pip (below, for apt and poetry), to improve +# rebuild speeds on slow connections. +# We install poetry as --user so that it doesn't end up in the system-wide python +# installation. That gets copied later into the runtime image. +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --user poetry==1.1.12 + +# install the OS build deps RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ @@ -45,33 +58,37 @@ RUN \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* -# Copy just what we need to pip install -COPY MANIFEST.in README.rst setup.py /synapse/ -COPY synapse/__init__.py /synapse/synapse/__init__.py -COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py +WORKDIR /synapse + +# Copy just what we need to run `poetry install` +COPY pyproject.toml poetry.lock README.rst /synapse/ + +# Install to the Python installation which hosts `pip`. In this case, it's the system +# Python. +ENV POETRY_VIRTUALENVS_IN_PROJECT=true \ + POETRY_VIRTUALENVS_CREATE=true \ + POETRY_HOME=/opt/poetry # To speed up rebuilds, install all of the dependencies before we copy over -# the whole synapse project so that we this layer in the Docker cache can be +# the whole synapse project, so that this layer in the Docker cache can be # used while you develop on the source -# -# This is aiming at installing the `install_requires` and `extras_require` from `setup.py` -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --prefix="/install" --no-warn-script-location \ - /synapse[all] +RUN --mount=type=cache,target=/opt/poetry/artifacts \ + --mount=type=cache,target=/opt/poetry/.cache/pypoetry/cache \ + /root/.local/bin/poetry install --no-dev --no-root --no-interaction --no-ansi --extras all -# Copy over the rest of the project +# Copy over the synapse source code. COPY synapse /synapse/synapse/ -# Install the synapse package itself and all of its children packages. -# -# This is aiming at installing only the `packages=find_packages(...)` from `setup.py -RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse +# Install the synapse package itself, by omitting the --no-root argument +RUN --mount=type=cache,target=/opt/poetry/artifacts \ + --mount=type=cache,target=/opt/poetry/cache \ + /root/.local/bin/poetry install --no-dev --no-interaction --no-ansi --extras all ### ### Stage 1: runtime ### -FROM docker.io/python:${PYTHON_VERSION}-slim +FROM base LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse' LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md' @@ -93,7 +110,7 @@ RUN \ openssl \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /install /usr/local +COPY --from=builder /synapse/ /synapse COPY ./docker/start.py /start.py COPY ./docker/conf /conf diff --git a/docker/start.py b/docker/start.py index ec9eeb49ae99..3de2635eb16a 100755 --- a/docker/start.py +++ b/docker/start.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python +#!/synapse/.venv/bin/python import codecs import glob @@ -108,7 +108,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership): # Hopefully we already have a signing key, but generate one if not. args = [ - "python", + sys.executable, "-m", "synapse.app.homeserver", "--config-path", @@ -158,7 +158,7 @@ def run_generate_config(environ, ownership): # generate the main config file, and a signing key. args = [ - "python", + sys.executable, "-m", "synapse.app.homeserver", "--server-name", @@ -175,7 +175,7 @@ def run_generate_config(environ, ownership): "--open-private-ports", ] # log("running %s" % (args, )) - os.execv("/usr/local/bin/python", args) + os.execv(sys.executable, args) def main(args, environ): @@ -254,12 +254,12 @@ def main(args, environ): log("Starting synapse with args " + " ".join(args)) - args = ["python"] + args + args = [sys.executable] + args if ownership is not None: args = ["gosu", ownership] + args os.execve("/usr/sbin/gosu", args, environ) else: - os.execve("/usr/local/bin/python", args, environ) + os.execve(sys.executable, args, environ) if __name__ == "__main__": From 0a28d77c95379e553e3893254acec277fcd2b408 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 5 Apr 2022 19:30:29 +0100 Subject: [PATCH 02/12] Changelog --- changelog.d/12385.docker | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12385.docker diff --git a/changelog.d/12385.docker b/changelog.d/12385.docker new file mode 100644 index 000000000000..abe2127ea0ff --- /dev/null +++ b/changelog.d/12385.docker @@ -0,0 +1 @@ +Bundle locked versions of dependencies into the Docker image. \ No newline at end of file From e741767046d736163ba9f6fe6669b61de6de945e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Apr 2022 12:18:00 +0100 Subject: [PATCH 03/12] Don't use the `base` image --- docker/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fc243053789d..ec295ccfa61a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,11 +16,10 @@ ARG PYTHON_VERSION=3.9 -FROM docker.io/python:${PYTHON_VERSION}-slim as base - ### ### Stage 0: builder ### +FROM docker.io/python:${PYTHON_VERSION}-slim as builder # Irritatingly, there is no blessed guide on how to distribute an application with its # poetry-managed environment in a docker image. For a while, @@ -29,7 +28,6 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as base # This is inspired from: # https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865 # https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc -FROM base as builder # RUN --mount is specific to buildkit and is documented at # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. @@ -88,7 +86,7 @@ RUN --mount=type=cache,target=/opt/poetry/artifacts \ ### Stage 1: runtime ### -FROM base +FROM docker.io/python:${PYTHON_VERSION}-slim LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse' LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md' From 92c9a60bd4b28ab9ba3c4747fe769c475fcd0665 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Apr 2022 12:42:34 +0100 Subject: [PATCH 04/12] Install via requirements.txt --- docker/Dockerfile | 30 +++++++++++++----------------- docker/start.py | 2 +- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ec295ccfa61a..ed1dc9edfbf7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,10 +22,11 @@ ARG PYTHON_VERSION=3.9 FROM docker.io/python:${PYTHON_VERSION}-slim as builder # Irritatingly, there is no blessed guide on how to distribute an application with its -# poetry-managed environment in a docker image. For a while, -# `poetry export | pip install -r /dev/stdin` seemed plausible but is limited by bugs +# poetry-managed environment in a docker image. We have opted for +# `poetry export | pip install -r /dev/stdin`, but there are known bugs in # in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released. -# This is inspired from: +# In case we get bitten by those bugs in the future, the recommendations here might +# be useful: # https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865 # https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc @@ -61,26 +62,21 @@ WORKDIR /synapse # Copy just what we need to run `poetry install` COPY pyproject.toml poetry.lock README.rst /synapse/ -# Install to the Python installation which hosts `pip`. In this case, it's the system -# Python. -ENV POETRY_VIRTUALENVS_IN_PROJECT=true \ - POETRY_VIRTUALENVS_CREATE=true \ - POETRY_HOME=/opt/poetry +RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt # To speed up rebuilds, install all of the dependencies before we copy over # the whole synapse project, so that this layer in the Docker cache can be # used while you develop on the source -RUN --mount=type=cache,target=/opt/poetry/artifacts \ - --mount=type=cache,target=/opt/poetry/.cache/pypoetry/cache \ - /root/.local/bin/poetry install --no-dev --no-root --no-interaction --no-ansi --extras all +# +# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml. +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --prefix="/install" --no-warn-script-location -r /synapse/requirements.txt -# Copy over the synapse source code. +# Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ -# Install the synapse package itself, by omitting the --no-root argument -RUN --mount=type=cache,target=/opt/poetry/artifacts \ - --mount=type=cache,target=/opt/poetry/cache \ - /root/.local/bin/poetry install --no-dev --no-interaction --no-ansi --extras all +# Install the synapse package itself. +RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse ### ### Stage 1: runtime @@ -108,7 +104,7 @@ RUN \ openssl \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /synapse/ /synapse +COPY --from=builder /install /usr/local COPY ./docker/start.py /start.py COPY ./docker/conf /conf diff --git a/docker/start.py b/docker/start.py index 3de2635eb16a..ac62bbc8baf9 100755 --- a/docker/start.py +++ b/docker/start.py @@ -1,4 +1,4 @@ -#!/synapse/.venv/bin/python +#!/usr/local/bin/python import codecs import glob From a1d7ba9885ab9c4fabfcd8263de27802e2c5b7d2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Apr 2022 15:28:47 +0100 Subject: [PATCH 05/12] Oh yeah, we're still using setuptools for wheels :( --- .dockerignore | 4 ++++ docker/Dockerfile | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 7809863ef328..a236760cf1fd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,8 @@ !pyproject.toml !poetry.lock +# TODO: remove these once we have moved over to using poetry-core in pyproject.toml +!MANIFEST.in +!setup.py + **/__pycache__ diff --git a/docker/Dockerfile b/docker/Dockerfile index ed1dc9edfbf7..64f99308579a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -59,7 +59,7 @@ RUN \ WORKDIR /synapse -# Copy just what we need to run `poetry install` +# Copy just what we need to run `poetry export`... COPY pyproject.toml poetry.lock README.rst /synapse/ RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt @@ -74,6 +74,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ +# ... and the setuptools metadata needed to `pip install`. +# TODO: once pyproject.toml declares poetry-core as its build system, +# we can remove this COPY statement. +COPY setup.py MANIFEST.in /synapse/ # Install the synapse package itself. RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse From 87ee7c21b93bac8fd38b3d97c1f45a5dcff67fd5 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Apr 2022 15:53:23 +0100 Subject: [PATCH 06/12] Try poetry 1.2.0b1 --- docker/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 64f99308579a..a9304c076dc4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,9 +35,11 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as builder # Here we use it to set up a cache for pip (below, for apt and poetry), to improve # rebuild speeds on slow connections. # We install poetry as --user so that it doesn't end up in the system-wide python -# installation. That gets copied later into the runtime image. +# installation. +# We use the unstable 1.2.0b1 version instead of our usual 1.1.12 to incorporate +# fixes to some bugs in `poetry export`. RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --user poetry==1.1.12 + pip install --user poetry==1.2.0b1 # install the OS build deps RUN \ From 5bc8eac555fe5c4bdd44b83a67236b1f53d10773 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Apr 2022 16:35:32 +0100 Subject: [PATCH 07/12] Try a three-stage build --- docker/Dockerfile | 52 +++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a9304c076dc4..047b63297228 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,13 +14,6 @@ # DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 . # -ARG PYTHON_VERSION=3.9 - -### -### Stage 0: builder -### -FROM docker.io/python:${PYTHON_VERSION}-slim as builder - # Irritatingly, there is no blessed guide on how to distribute an application with its # poetry-managed environment in a docker image. We have opted for # `poetry export | pip install -r /dev/stdin`, but there are known bugs in @@ -30,17 +23,38 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as builder # https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865 # https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc + + +ARG PYTHON_VERSION=3.9 + +### +### Stage 0: generate requirements.txt +### +FROM docker.io/python:${PYTHON_VERSION}-slim as requirements + # RUN --mount is specific to buildkit and is documented at # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. -# Here we use it to set up a cache for pip (below, for apt and poetry), to improve +# Here we use it to set up a cache for pip (and below for apt), to improve # rebuild speeds on slow connections. -# We install poetry as --user so that it doesn't end up in the system-wide python -# installation. +# We install poetry in its own build stage to avoid its dependencies conflicting with +# synapse's dependencies. # We use the unstable 1.2.0b1 version instead of our usual 1.1.12 to incorporate # fixes to some bugs in `poetry export`. RUN --mount=type=cache,target=/root/.cache/pip \ pip install --user poetry==1.2.0b1 +WORKDIR /synapse + +# Copy just what we need to run `poetry export`... +COPY pyproject.toml poetry.lock README.rst /synapse/ + +RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt + +### +### Stage 1: builder +### +FROM docker.io/python:${PYTHON_VERSION}-slim as builder + # install the OS build deps RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ @@ -59,33 +73,27 @@ RUN \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* -WORKDIR /synapse - -# Copy just what we need to run `poetry export`... -COPY pyproject.toml poetry.lock README.rst /synapse/ - -RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt - # To speed up rebuilds, install all of the dependencies before we copy over # the whole synapse project, so that this layer in the Docker cache can be # used while you develop on the source # # This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml. +COPY --from=requirements /synapse/requirements.txt /synapse/ RUN --mount=type=cache,target=/root/.cache/pip \ pip install --prefix="/install" --no-warn-script-location -r /synapse/requirements.txt # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ -# ... and the setuptools metadata needed to `pip install`. -# TODO: once pyproject.toml declares poetry-core as its build system, -# we can remove this COPY statement. -COPY setup.py MANIFEST.in /synapse/ +# ... and what we need to `pip install`. +# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy +# pyproject.toml here, ditching setup.py and MANIFEST.in. +COPY setup.py MANIFEST.in README.rst /synapse/ # Install the synapse package itself. RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse ### -### Stage 1: runtime +### Stage 2: runtime ### FROM docker.io/python:${PYTHON_VERSION}-slim From 98e8b39f5a4530f5b4abd53c077d7c28f785c889 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Apr 2022 16:36:45 +0100 Subject: [PATCH 08/12] Desperation: ditch PyPy for a sensible export --- poetry.lock | 35 +++------------------------------- pyproject.toml | 6 ++---- synapse/python_dependencies.py | 2 -- 3 files changed, 5 insertions(+), 38 deletions(-) diff --git a/poetry.lock b/poetry.lock index 17430cbfa6c1..355c49ee0581 100644 --- a/poetry.lock +++ b/poetry.lock @@ -734,29 +734,6 @@ category = "main" optional = true python-versions = ">=3.6" -[[package]] -name = "psycopg2cffi" -version = "2.9.0" -description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=master" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -cffi = ">=1.0" -six = "*" - -[[package]] -name = "psycopg2cffi-compat" -version = "1.1" -description = "A Simple library to enable psycopg2 compatability" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -psycopg2 = "*" - [[package]] name = "pyasn1" version = "0.4.8" @@ -1576,13 +1553,13 @@ docs = ["sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] -all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis"] +all = ["matrix-synapse-ldap3", "psycopg2", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis"] cache_memory = ["Pympler"] jwt = ["pyjwt"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] oidc = ["authlib"] opentracing = ["jaeger-client", "opentracing"] -postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"] +postgres = ["psycopg2"] redis = ["txredisapi", "hiredis"] saml2 = ["pysaml2"] sentry = ["sentry-sdk"] @@ -1593,7 +1570,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "7ff6d982a9d6675cb595b216b23549ef1942d0e39cb91c97494ff6ed95a9e8d2" +content-hash = "351bc7d8214d3ccab4ae0608c83169af97f734476b6883c515bbf172f1fc7f77" [metadata.files] appdirs = [ @@ -2241,12 +2218,6 @@ psycopg2 = [ {file = "psycopg2-2.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:06f32425949bd5fe8f625c49f17ebb9784e1e4fe928b7cce72edc36fb68e4c0c"}, {file = "psycopg2-2.9.3.tar.gz", hash = "sha256:8e841d1bf3434da985cc5ef13e6f75c8981ced601fd70cc6bf33351b91562981"}, ] -psycopg2cffi = [ - {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, -] -psycopg2cffi-compat = [ - {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, -] pyasn1 = [ {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"}, diff --git a/pyproject.toml b/pyproject.toml index 92ea302b75d4..c78770510813 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,8 +176,6 @@ importlib_metadata = { version = ">=1.4", python = "<3.8" } # --------------------- matrix-synapse-ldap3 = { version = ">=0.1", optional = true } psycopg2 = { version = ">=2.8", markers = "platform_python_implementation != 'PyPy'", optional = true } -psycopg2cffi = { version = ">=2.8", markers = "platform_python_implementation == 'PyPy'", optional = true } -psycopg2cffi-compat = { version = "==1.1", markers = "platform_python_implementation == 'PyPy'", optional = true } pysaml2 = { version = ">=4.5.0", optional = true } authlib = { version = ">=0.14.0", optional = true } # systemd-python is necessary for logging to the systemd journal via @@ -199,7 +197,7 @@ parameterized = { version = ">=0.7.4", optional = true } # NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified # twice: once here, and once in the `all` extra. matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] -postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"] +postgres = ["psycopg2"] saml2 = ["pysaml2"] oidc = ["authlib"] # systemd-python is necessary for logging to the systemd journal via @@ -294,4 +292,4 @@ towncrier = ">=18.6.0rc1" [build-system] requires = ["setuptools"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index cd68aa362e83..b81b37b2f300 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -99,8 +99,6 @@ "postgres": [ # we use execute_values with the fetch param, which arrived in psycopg 2.8. "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", - "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", - "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", ], "saml2": [ "pysaml2>=4.5.0", From efeb99d63a732ff02a075049dd55cf265a4403ae Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Apr 2022 18:23:48 +0100 Subject: [PATCH 09/12] Revert "Desperation: ditch PyPy for a sensible export" This reverts commit 98e8b39f5a4530f5b4abd53c077d7c28f785c889. --- poetry.lock | 35 +++++++++++++++++++++++++++++++--- pyproject.toml | 6 ++++-- synapse/python_dependencies.py | 2 ++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 355c49ee0581..17430cbfa6c1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -734,6 +734,29 @@ category = "main" optional = true python-versions = ">=3.6" +[[package]] +name = "psycopg2cffi" +version = "2.9.0" +description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=master" +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +cffi = ">=1.0" +six = "*" + +[[package]] +name = "psycopg2cffi-compat" +version = "1.1" +description = "A Simple library to enable psycopg2 compatability" +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +psycopg2 = "*" + [[package]] name = "pyasn1" version = "0.4.8" @@ -1553,13 +1576,13 @@ docs = ["sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] -all = ["matrix-synapse-ldap3", "psycopg2", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis"] +all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis"] cache_memory = ["Pympler"] jwt = ["pyjwt"] matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] oidc = ["authlib"] opentracing = ["jaeger-client", "opentracing"] -postgres = ["psycopg2"] +postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"] redis = ["txredisapi", "hiredis"] saml2 = ["pysaml2"] sentry = ["sentry-sdk"] @@ -1570,7 +1593,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "351bc7d8214d3ccab4ae0608c83169af97f734476b6883c515bbf172f1fc7f77" +content-hash = "7ff6d982a9d6675cb595b216b23549ef1942d0e39cb91c97494ff6ed95a9e8d2" [metadata.files] appdirs = [ @@ -2218,6 +2241,12 @@ psycopg2 = [ {file = "psycopg2-2.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:06f32425949bd5fe8f625c49f17ebb9784e1e4fe928b7cce72edc36fb68e4c0c"}, {file = "psycopg2-2.9.3.tar.gz", hash = "sha256:8e841d1bf3434da985cc5ef13e6f75c8981ced601fd70cc6bf33351b91562981"}, ] +psycopg2cffi = [ + {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, +] +psycopg2cffi-compat = [ + {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, +] pyasn1 = [ {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"}, diff --git a/pyproject.toml b/pyproject.toml index c78770510813..92ea302b75d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,6 +176,8 @@ importlib_metadata = { version = ">=1.4", python = "<3.8" } # --------------------- matrix-synapse-ldap3 = { version = ">=0.1", optional = true } psycopg2 = { version = ">=2.8", markers = "platform_python_implementation != 'PyPy'", optional = true } +psycopg2cffi = { version = ">=2.8", markers = "platform_python_implementation == 'PyPy'", optional = true } +psycopg2cffi-compat = { version = "==1.1", markers = "platform_python_implementation == 'PyPy'", optional = true } pysaml2 = { version = ">=4.5.0", optional = true } authlib = { version = ">=0.14.0", optional = true } # systemd-python is necessary for logging to the systemd journal via @@ -197,7 +199,7 @@ parameterized = { version = ">=0.7.4", optional = true } # NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified # twice: once here, and once in the `all` extra. matrix-synapse-ldap3 = ["matrix-synapse-ldap3"] -postgres = ["psycopg2"] +postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"] saml2 = ["pysaml2"] oidc = ["authlib"] # systemd-python is necessary for logging to the systemd journal via @@ -292,4 +294,4 @@ towncrier = ">=18.6.0rc1" [build-system] requires = ["setuptools"] -build-backend = "setuptools.build_meta" +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index b81b37b2f300..cd68aa362e83 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -99,6 +99,8 @@ "postgres": [ # we use execute_values with the fetch param, which arrived in psycopg 2.8. "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", + "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", + "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", ], "saml2": [ "pysaml2>=4.5.0", From 43b9b3e3f2678d55ac1e7f4565f6e3eb2e8a5754 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Apr 2022 18:25:01 +0100 Subject: [PATCH 10/12] Use poetry git --- docker/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 047b63297228..374e651a00d9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -32,6 +32,12 @@ ARG PYTHON_VERSION=3.9 ### FROM docker.io/python:${PYTHON_VERSION}-slim as requirements +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y git \ + && rm -rf /var/lib/apt/lists/* + # RUN --mount is specific to buildkit and is documented at # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. # Here we use it to set up a cache for pip (and below for apt), to improve @@ -41,7 +47,7 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as requirements # We use the unstable 1.2.0b1 version instead of our usual 1.1.12 to incorporate # fixes to some bugs in `poetry export`. RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --user poetry==1.2.0b1 + pip install --user git+https://github.com/python-poetry/poetry.git@3780a727f3bfebed99c68c262c53348fa12e7eb7 WORKDIR /synapse From 432adf7ee1de25e98bcfd0d285d70aaa3979203b Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 7 Apr 2022 12:10:42 +0100 Subject: [PATCH 11/12] Move RUN --mount comment --- docker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 374e651a00d9..0a3a5037230d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -32,16 +32,16 @@ ARG PYTHON_VERSION=3.9 ### FROM docker.io/python:${PYTHON_VERSION}-slim as requirements +# RUN --mount is specific to buildkit and is documented at +# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. +# Here we use it to set up a cache for apt (and below for pip), to improve +# rebuild speeds on slow connections. RUN \ --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update && apt-get install -y git \ && rm -rf /var/lib/apt/lists/* -# RUN --mount is specific to buildkit and is documented at -# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. -# Here we use it to set up a cache for pip (and below for apt), to improve -# rebuild speeds on slow connections. # We install poetry in its own build stage to avoid its dependencies conflicting with # synapse's dependencies. # We use the unstable 1.2.0b1 version instead of our usual 1.1.12 to incorporate From 1a81c24209e84f817e8b8d967e3bfa06c38108fd Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 7 Apr 2022 12:12:30 +0100 Subject: [PATCH 12/12] Use the specific poetry commit which fixes stuff --- docker/Dockerfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0a3a5037230d..6009da7db7eb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,10 +44,17 @@ RUN \ # We install poetry in its own build stage to avoid its dependencies conflicting with # synapse's dependencies. -# We use the unstable 1.2.0b1 version instead of our usual 1.1.12 to incorporate -# fixes to some bugs in `poetry export`. +# We use a specific commit from poetry's master branch instead of our usual 1.1.12, +# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to +# https://github.com/python-poetry/poetry/pull/5156 and +# https://github.com/python-poetry/poetry/issues/5141 ; +# without it, we generate a requirements.txt with incorrect environment markers, +# which causes necessary packages to be omitted when we `pip install`. +# +# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also +# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export). RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --user git+https://github.com/python-poetry/poetry.git@3780a727f3bfebed99c68c262c53348fa12e7eb7 + pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5 WORKDIR /synapse