Allow for private CA on the Agent side

PBENCH-1209

The staging server and local development servers now use HTTPS with certs
signed by a private pbench CA. An HTTPS connection can't be validated without
a reference to this CA.

The primary change here is in the `contrib/containerized-pbench/pbench` script
which now looks for a private CA definition, maps the CA bundle file into the
container, and defines `REQUESTS_CA_BUNDLE` within the container.

In an attempt to handle RPM installs, there's also logic to support a new
`[results]` section `pbench_ca` configuration variable to define a CA path
that will be used to verify the `PUT` to a server.

Note, this isn't being used for the `--relay` path, as that's currently not
using `https` and we'll need to figure out how we want to configure this in
the future.

agent/config/pbench-agent-default.cfg

@@ -21,6 +21,7 @@ ssh_opts = -o BatchMode=yes -o StrictHostKeyChecking=no
 api_version = 1
 rest_endpoint = api/v%(api_version)s
 server_rest_url = https://%(pbench_web_server)s/%(rest_endpoint)s
+#server_ca =
 
 [pbench/tools]
 light-tool-set = vmstat

agent/config/pbench-agent.cfg

@@ -6,3 +6,6 @@ pbench_web_server = pbench.example.com
 [config]
 path = %(pbench_install_dir)s/config
 files = pbench-agent-default.cfg
+
+[results]
+#server_ca =

contrib/containerized-pbench/pbench

@@ -6,11 +6,12 @@
 # container, without needing to install the Agent on the host system.
 #
 # Invocation options are provided as environment variables:
-#    PB_AGENT_IMAGE_NAME:  the full image name for the containerized Pbench Agent
-#    _PBENCH_AGENT_CONFIG:  the location of the Pbench Agent configuration file
-#    PB_AGENT_RUN_DIR:  the directory for use as the Pbench Agent "run directory"
-#    PB_AGENT_SERVER_LOC:  the host and port for the Pbench Server
-#    PB_AGENT_PODMAN_OPTIONS:  Additional options to be supplied to Podman run
+#    PB_AGENT_IMAGE_NAME: the full image name for the containerized Pbench Agent
+#    _PBENCH_AGENT_CONFIG: the location of the Pbench Agent configuration file
+#    PB_AGENT_RUN_DIR: the directory for use as the Pbench Agent "run directory"
+#    PB_AGENT_SERVER_LOC: the host and port for the Pbench Server
+#    PB_AGENT_CA: a CA bundle to verify Pbench Server PUTs
+#    PB_AGENT_PODMAN_OPTIONS: Additional options to be supplied to Podman run
 #
 # In all cases, reasonable defaults are supplied if the environment variables
 # are not defined.
@@ -28,6 +29,7 @@ image_name=${PB_AGENT_IMAGE_NAME:-quay.io/pbench/pbench-agent-all-centos-8:main}
 config_file=${_PBENCH_AGENT_CONFIG:-${HOME}/.config/pbench/pbench-agent.cfg}
 pbench_run_dir=${PB_AGENT_RUN_DIR:-/var/tmp/${USER}/pbench-agent/run}
 pbench_server=${PB_AGENT_SERVER_LOC}
+pbench_ca=$(readlink -f ${PB_AGENT_CA:-${REQUESTS_CA_BUNDLE}})
 other_options=${PB_AGENT_PODMAN_OPTIONS}
 
 if [[ $# == 0 || $1 == "help" || $1 == "-h" || $1 == "--help" ]]; then
@@ -52,12 +54,18 @@ elif [[ -n "${pbench_server}" ]]; then
 		[config]
 		path = %(pbench_install_dir)s/config
 		files = pbench-agent-default.cfg
+		[results]
+		pbench_ca = ${pbench_ca}
 		EOF
 else
     echo "Warning:  the Pbench Agent config file (e.g., ${config_file}) is missing or inaccessible -- using default configuration." >&2
 fi
 
 mkdir -p ${pbench_run_dir}
+if [[ -f "${pbench_ca}" ]]; then
+    other_options="-v ${pbench_ca}:/etc/pki/tls/certs/pbench_CA.crt:z ${other_options}"
+    other_options="-e REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/pbench_CA.crt ${other_options}"
+fi
 other_options="-v ${pbench_run_dir}:/var/lib/pbench-agent:z ${other_options}"
 
 podman run \

lib/pbench/agent/results.py

@@ -406,8 +406,13 @@ def __init__(
         if server:
             path = config.get("results", "rest_endpoint")
             uri = f"{server}/{path}"
+            self.ca = True
         else:
             uri = config.get("results", "server_rest_url")
+
+            # If the "server_ca" config variable isn't defined, we expect to verify
+            # using a registered CA or via REQUESTS_CA_BUNDLE environment variable.
+            self.ca = config.get("results", "server_ca", fallback=True)
         self.uri = f"{uri}/upload/{{name}}"
         self.headers.update({"Authorization": f"Bearer {token}"})
 
@@ -430,7 +435,11 @@ def push(self, tarball: Path, tarball_md5: str) -> requests.Response:
         tar_uri = self.uri.format(name=tarball.name)
         with tarball.open("rb") as f:
             return requests.put(
-                tar_uri, data=f, headers=self.headers, params=self.params
+                tar_uri,
+                data=f,
+                headers=self.headers,
+                params=self.params,
+                verify=self.ca,
             )