Add IT tests to test installation according to docs

.ci/build.sh

@@ -24,6 +24,11 @@ set -o pipefail
 # fail on any unset environment variables
 set -u
 
+function update_pyenv {
+  # need to have the latest pyenv version to ensure latest patch releases are installable
+  cd $HOME/.pyenv/plugins/python-build/../.. && git pull origin master --rebase && cd -
+}
+
 function build {
   export THESPLOG_FILE="${THESPLOG_FILE:-${RALLY_HOME}/.rally/logs/actor-system-internal.log}"
   # this value is in bytes, the default is 50kB. We increase it to 200kiB.
@@ -37,6 +42,7 @@ function build {
   export PATH="$HOME/.pyenv/bin:$PATH"
   export TERM=dumb
   export LC_ALL=en_US.UTF-8
+  update_pyenv
   eval "$(pyenv init -)"
   eval "$(pyenv virtualenv-init -)"
 

.ci/variables.json

@@ -1,6 +1,6 @@
 {
   "python_versions": {
-    "PY38": "3.8.0",
-    "PY39": "3.9.0"
+    "PY38": "3.8.8",
+    "PY39": "3.9.2"
   }
 }

Makefile

@@ -21,8 +21,9 @@ PYENV_REGEX = .pyenv/shims
 PY_BIN = python3
 # https://github.com/pypa/pip/issues/5599
 PIP_WRAPPER = $(PY_BIN) -m pip
-PY38 = $(shell jq '.python_versions.PY38' .ci/variables.json)
-PY39 = $(shell jq '.python_versions.PY39' .ci/variables.json)
+export PY38 = $(shell jq -r '.python_versions.PY38' .ci/variables.json)
+export PY39 = $(shell jq -r '.python_versions.PY39' .ci/variables.json)
+export MIN_PY_VER = $(PY38)
 VENV_NAME ?= .venv
 VENV_ACTIVATE_FILE = $(VENV_NAME)/bin/activate
 VENV_ACTIVATE = . $(VENV_ACTIVATE_FILE)

docs/conf.py

@@ -18,6 +18,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import json
 import os
 from datetime import date
 from os.path import join, dirname
@@ -40,13 +41,35 @@
 master_doc = 'index'
 language = None
 
-year = date.today().year
 
-rst_prolog = '''
-.. |year| replace:: {0}
-'''.format(year)
+def read_min_python_version():
+    return os.environ["MIN_PY_VER"]
+
+
+GLOBAL_SUBSTITUTIONS = {
+    "{MIN_PY_VER}": read_min_python_version()
+}
+
+
+# inspiration from https://github.com/sphinx-doc/sphinx/issues/4054#issuecomment-329097229
+def replace_globals(app, docname, source):
+    tmp_source = source[0]
+    for k, v in GLOBAL_SUBSTITUTIONS.items():
+        tmp_source = tmp_source.replace(k, v)
+    source[0] = tmp_source
+
+
+def setup(app):
+    app.connect("source-read", replace_globals)
 
 
+year = date.today().year
+
+rst_prolog = f"""
+.. |year| replace:: {year}
+.. |MIN_PY_VER| replace:: {read_min_python_version()}
+"""
+
 # General information about the project.
 project = "Rally"
 copyright = "%i, Elasticsearch B.V." % year
@@ -58,8 +81,9 @@
 #
 # The short X.Y version.
 
-
 # development versions always have the suffix '.dev0'
+
+
 def read_version(full_version=True):
     with open(join(dirname(__file__), os.pardir, "version.txt")) as f:
         raw_version = f.read().strip()

docs/install.rst

@@ -27,11 +27,11 @@ We recommend to use `pyenv <https://github.com/pyenv/pyenv>`_ to manage installa
 Once ``pyenv`` is installed, install a compatible Python version::
 
     # Install Python
-    pyenv install 3.8.7
+    pyenv install {MIN_PY_VER}
 
     # select that version for the current user
     # see https://github.com/pyenv/pyenv/blob/master/COMMANDS.md#pyenv-global for details
-    pyenv global 3.8.7
+    pyenv global {MIN_PY_VER}
 
     # Upgrade pip
     python3 -m pip install --user --upgrade pip

it/__init__.py

@@ -90,6 +90,23 @@ def race(cfg, command_line):
     return esrally(cfg, f"race {command_line} --kill-running-processes --on-error='abort' --enable-assertions")
 
 
+def shell_cmd(command_line):
+    """
+    Executes a given command_line in a subshell.
+
+    :param command_line: (str) The command to execute
+    :return: (int) the exit code
+    """
+
+    return os.system(command_line)
+
+
+def command_in_docker(command_line, python_version):
+    docker_command = f"docker run --rm -v {ROOT_DIR}:/rally_ro:ro python:{python_version} bash -c '{command_line}'"
+
+    return shell_cmd(docker_command)
+
+
 def wait_until_port_is_free(port_number=39200, timeout=120):
     start = time.perf_counter()
     end = start + timeout

it/distribution_test.py

@@ -28,7 +28,7 @@ def test_tar_distributions(cfg):
         for track in it.TRACKS:
             it.wait_until_port_is_free()
             assert it.race(cfg, f"--distribution-version=\"{dist}\" --track=\"{track}\" "
-                                   f"--test-mode --car=4gheap") == 0
+                                f"--test-mode --car=4gheap") == 0
 
 
 @it.random_rally_config
@@ -37,8 +37,8 @@ def test_docker_distribution(cfg):
     dist = it.DISTRIBUTIONS[-1]
     it.wait_until_port_is_free(port_number=19200)
     assert it.race(cfg, f"--pipeline=\"docker\" --distribution-version=\"{dist}\" "
-                           f"--track=\"geonames\" --challenge=\"append-no-conflicts-index-only\" --test-mode "
-                           f"--car=4gheap --target-hosts=127.0.0.1:19200") == 0
+                        f"--track=\"geonames\" --challenge=\"append-no-conflicts-index-only\" --test-mode "
+                        f"--car=4gheap --target-hosts=127.0.0.1:19200") == 0
 
 
 @pytest.fixture(scope="module")

it/installation_test.py

@@ -0,0 +1,74 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+These tests ensure the validatity of Rally installation instructions (as shown in docs)
+"""
+
+import json
+import os
+import warnings
+
+import it
+
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+_CI_VARS = ".ci/variables.json"
+
+# normally set by Makefile, be lenient to allow running from IDE
+MIN_PY_VER = os.environ.get("MIN_PY_VER")
+if not MIN_PY_VER:
+    with open(os.path.join(ROOT_DIR, _CI_VARS), "rt") as fp:
+        try:
+            MIN_PY_VER = json.load(fp)["python_versions"]["PY38"]
+            warnings.warn(
+                f"Environment variable [MIN_PY_VER] is missing. You should fix this problem "
+                f"if you are running these tests "
+                f"on anything else than through your IDE. Defaulting to [{MIN_PY_VER}].",
+                category=RuntimeWarning
+            )
+        except KeyError:
+            raise EnvironmentError(f"Installation tests require [python_versions.PY38] key in [{_CI_VARS}] or"
+                                   f"the [MIN_PY_VER] environment variable set.")
+
+
+def test_installs_inside_venv():
+    # as in the VirtualEnv Install section of install.rst
+    commands = (
+        "cp -a /rally_ro /rally &&"
+        "python3 -mvenv .venv &&"
+        "source .venv/bin/activate &&"
+        "cd /rally &&"
+        "python3 -m pip install --upgrade pip &&"
+        "python3 -m pip install -e . &&"
+        "esrally list tracks"
+    )
+
+    assert it.command_in_docker(commands, python_version=MIN_PY_VER) == 0
+
+
+def test_local_installation():
+    # as in the Installing Rally install.rst
+    commands = (
+        "cp -a /rally_ro /rally &&"
+        "cd /rally &&"
+        "export PATH=$PATH:~/.local/bin &&"
+        "python3 -m pip install --user --upgrade pip &&"
+        "python3 -m pip install --user -e . &&"
+        "esrally list tracks"
+    )
+
+    assert it.command_in_docker(commands, python_version=MIN_PY_VER) == 0

tox.ini

@@ -22,6 +22,7 @@ passenv =
     JAVA13_HOME
     JAVA14_HOME
     JAVA15_HOME
+    MIN_PY_VER
     RALLY_HOME
     SSH_AUTH_SOCK
     THESPLOG_FILE