Merge branch 'main' into frontend-refactor

* main:
  Added missing network-intercepts migration
  Schema checker improvements (#395)
  Bump fastapi from 0.103.0 to 0.103.1 (#448)
  Bump pytest from 7.4.0 to 7.4.1 (#449)
  Added python and system packages listing (#430)
  added workflow for building and pushing containers from docker/auxili… (#394)

.github/workflows/build-and-push-containers.yml

@@ -0,0 +1,36 @@
+name: Build and Push Containers
+on:
+  pull_request:
+    types:
+      - closed
+    paths:
+      - 'docker/auxiliary-containers/**/Dockerfile'
+
+  workflow_dispatch:
+
+jobs:
+  build-and-push-containers:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+        ## This is needed for multi-architecture builds
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Registry
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and Push Containers
+        run: bash ./docker/auxiliary-containers/build-containers.sh

docker/auxiliary-containers/build-containers.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# The names of the folders within "auxiliary-containers" must match the repository name in dockerhub!
+
+# Get the list of subdirectories within "auxiliary-containers" directory containing a Dockerfile
+subdirs=($(find ./docker/auxiliary-containers -type f -name 'Dockerfile' -exec dirname {} \;))
+
+# Loop through each subdirectory, build and push the Docker image
+for subdir in "${subdirs[@]}"; do
+  folder=$(basename "${subdir}")
+  docker buildx build \
+    --push \
+    --tag "greencoding/${folder}:latest" \
+    --platform linux/amd64,linux/arm64 \
+    "${subdir}"
+done

docker/auxiliary-containers/gcb_playwright/Dockerfile

@@ -0,0 +1,13 @@
+FROM mcr.microsoft.com/playwright/python:v1.35.0-jammy
+
+# Install dependencies
+RUN apt-get update && apt-get install -y  curl  wget  gnupg  && rm -rf /var/lib/apt/lists/*
+
+# Install Playwright
+RUN pip install playwright==1.35.0
+
+# Set up Playwright dependencies for Chromium, Firefox and Webkit
+RUN playwright install
+RUN playwright install-deps
+
+CMD ["/bin/bash"]

docker/requirements.txt

@@ -1,6 +1,6 @@
 gunicorn==21.2.0
 psycopg[binary]==3.1.10
-fastapi==0.103.0
+fastapi==0.103.1
 uvicorn[standard]==0.23.2
 pandas==2.1.0
 PyYAML==6.0.1

lib/hardware_info.py

@@ -4,9 +4,10 @@
 '''
 import re
 import os
-import subprocess
 import platform
 import pprint
+import subprocess
+import sys
 
 REGEX_PARAMS = re.MULTILINE | re.IGNORECASE
 
@@ -77,6 +78,9 @@ def read_directory_recursive(directory):
     [rpwr, 'Hardware Model', '/usr/bin/hostnamectl', r'Hardware Model:\s*(?P<o>.*)'],
     [rpwr, 'Docker Info', 'docker info', r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
     [rpwr, 'Docker Version', 'docker version', r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
+    [rpwr, 'Docker Containers', 'docker ps -a', r'(?P<o>.*)'],
+    [rpwr, 'Installed System Packages', 'if [ -f /etc/lsb-release ]; then dpkg -l ; elif [ -f /etc/redhat-release ]; then dnf list installed ; fi', r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
+    [rpwr, 'Installed Python Packages', f"{sys.executable} -m pip freeze", r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
     [rpwr, 'Processes', '/usr/bin/ps -aux', r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
     [
         rpwrs,
@@ -106,6 +110,7 @@ def read_directory_recursive(directory):
     [rpwr, 'Uname', 'uname -a', r'(?P<o>.*)'],
     [rpwr, 'Docker Info', 'docker info', r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
     [rpwr, 'Docker Version', 'docker version', r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
+    [rpwr, 'Docker Containers', 'docker ps -a', r'(?P<o>.*)'],
     [rpwr, 'Processes', '/bin/ps -ax', r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
     [rpwr, 'Network Interfaces', 'ifconfig | grep -E "flags|ether"', r'(?P<o>.*)', re.IGNORECASE | re.DOTALL],
 

lib/schema_checker.py

@@ -2,7 +2,7 @@
 import string
 import re
 from schema import Schema, SchemaError, Optional, Or, Use
-# https://docs.green-coding.berlin/docs/measuring/usage-scenario/
+#
 # networks documentation is different than what i see in the wild!
     # name: str
     # also isn't networks optional?
@@ -67,6 +67,19 @@ def valid_service_types(self, value):
             raise SchemaError(f"{value} is not 'container'")
         return value
 
+    def validate_networks_no_invalid_chars(self, networks):
+        if isinstance(networks, list):
+            for item in networks:
+                if item is not None:
+                    self.contains_no_invalid_chars(item)
+        elif isinstance(networks, dict):
+            for key, value in networks.items():
+                self.contains_no_invalid_chars(key)
+                if value is not None:
+                    self.contains_no_invalid_chars(value)
+        else:
+            raise SchemaError("'networks' should be a list or a dictionary")
+
 
     def check_usage_scenario(self, usage_scenario):
         # Anything with Optional() is not needed, but if it exists must conform to the definition specified
@@ -75,9 +88,7 @@ def check_usage_scenario(self, usage_scenario):
             "author": str,
             "description": str,
 
-            Optional("networks"): {
-               Use(self.contains_no_invalid_chars): None
-            },
+            Optional("networks"): Or(list, dict),
 
             Optional("services"): {
                 Use(self.contains_no_invalid_chars): {
@@ -102,25 +113,35 @@ def check_usage_scenario(self, usage_scenario):
                     Optional("detach"): bool,
                     Optional("note"): str,
                     Optional("read-notes-stdout"): bool,
-                    Optional("ignore-errors"): bool
+                    Optional("ignore-errors"): bool,
+                    Optional("shell"): str,
+                    Optional("log-stdout"): bool,
+                    Optional("log-stderr"): bool,
                 }],
             }],
 
-            Optional("builds"): {
-                str:str
-            },
+            Optional("builds"): {str:str},
 
             Optional("compose-file"): Use(self.validate_compose_include)
         }, ignore_extra_keys=True)
 
+        # This check is necessary to do in a seperate pass. If tried to bake into the schema object above,
+        # it will not know how to handle the value passed when it could be either a dict or list
+        if 'networks' in usage_scenario:
+            self.validate_networks_no_invalid_chars(usage_scenario['networks'])
+
+        if "builds" not in usage_scenario and usage_scenario.get("services") is not None:
+            for service in usage_scenario["services"].values():
+                if "image" not in service:
+                    raise SchemaError("The 'image' key under services is required when 'builds' key is not present.")
+
         usage_scenario_schema.validate(usage_scenario)
 
 
 # if __name__ == '__main__':
 #     import yaml
 
 #     with open("test-file.yml", encoding='utf8') as f:
-#     # with open("test-file-2.yaml", encoding='utf8') as f:
 #         usage_scenario = yaml.safe_load(f)
 
 #     SchemaChecker = SchemaChecker(validate_compose_flag=True)

migrations/2023_09_01_network_intercepts.sql

@@ -0,0 +1,8 @@
+CREATE TABLE network_intercepts (
+    id SERIAL PRIMARY KEY,
+    project_id uuid NOT NULL REFERENCES projects(id) ON DELETE CASCADE ON UPDATE CASCADE ,
+    time bigint NOT NULL,
+    connection_type text NOT NULL,
+    protocol text NOT NULL,
+    created_at timestamp with time zone DEFAULT now()
+);

requirements-dev.txt

@@ -1,5 +1,5 @@
 -r requirements.txt
 pydantic==2.3.0
-pytest==7.4.0
+pytest==7.4.1
 requests==2.31.0
 pylint==2.17.5

test/data/usage_scenarios/schema_checker_valid.yml → test/data/usage_scenarios/schema_checker/schema_checker_invalid_image_builds.yml

@@ -1,15 +1,14 @@
 ---
 name: Test
 author: Dan Mateas
-description: test
+description: "test that image is required when build is not specified"
 
 networks: 
-  network-name:
+  networkname:
 
 services:
   test-container:
     type: container
-    image: gcb_stress
     build: .
 
 flow:

test/data/usage_scenarios/schema_checker/schema_checker_invalid_wrong_type.yml

@@ -0,0 +1,28 @@
+---
+name: Test
+author: Dan Mateas
+description: test
+
+networks: 
+  network-name:
+
+networks:
+  - network-a
+  - network-b
+
+services:
+  test-container:
+    type: container
+    image: gcb_stress
+    build: .
+
+flow:
+  - name: Stress
+    container: test-container
+    commands:
+      - type: console
+        command: stress-ng -c 1 -t 1 -q
+        note: Starting Stress
+        shell: bash
+        log-stdout: true
+        log-stderr: "no" # should throw error, not a bool

test/data/usage_scenarios/schema_checker/schema_checker_valid.yml

@@ -0,0 +1,28 @@
+---
+name: Test
+author: Dan Mateas
+description: test
+
+networks: 
+  network-name:
+  network-name-2:
+
+services:
+  test-container:
+    type: container
+    image: gcb_stress
+    build: .
+  test-container-2:
+    type: container
+    image: fizzbump
+
+flow:
+  - name: Stress
+    container: test-container
+    commands:
+      - type: console 
+        command: stress-ng -c 1 -t 1 -q 
+        note: Starting Stress
+        shell: bash 
+        log-stdout: true 
+        log-stderr: false

test/data/usage_scenarios/schema_checker/schema_checker_valid_network_as_keys.yml

@@ -0,0 +1,25 @@
+---
+name: Test
+author: Dan Mateas
+description: test
+
+networks: 
+  network-name:
+  network-name-2:
+
+services:
+  test-container:
+    type: container
+    image: gcb_stress
+    build: .
+
+flow:
+  - name: Stress
+    container: test-container
+    commands:
+      - type: console 
+        command: stress-ng -c 1 -t 1 -q 
+        note: Starting Stress
+        shell: bash 
+        log-stdout: true
+        log-stderr: false

test/data/usage_scenarios/schema_checker/schema_checker_valid_network_as_list.yml

@@ -0,0 +1,25 @@
+---
+name: Test
+author: Dan Mateas
+description: test
+
+networks:
+  - network-a
+  - network-b
+
+services:
+  test-container:
+    type: container
+    image: gcb_stress
+    build: .
+
+flow:
+  - name: Stress
+    container: test-container
+    commands:
+      - type: console 
+        command: stress-ng -c 1 -t 1 -q 
+        note: Starting Stress
+        shell: bash 
+        log-stdout: true 
+        log-stderr: false