Simplify logging

.github/workflows/test.yaml

@@ -12,15 +12,36 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      # Install dependencies if cache does not exist
       - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Install project
+        run: poetry install --no-interaction
+
+      - name: Run tests
         run: |
-          python -m pip install --upgrade pip
-          pip install pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Test with pytest
-        run: |
-          pytest
+          source .venv/bin/activate
+          pytest tests/
+          coverage report

README.md

@@ -1,43 +1,51 @@
 # phospho Python Client
 
-This is a Python client for phospho.
+Phospho is a platform to help you monitor. 
 
-For more information, see the docs at [docs.phospho.app](https://docs.phospho.app/).
+Read the docs at [docs.phospho.app](https://docs.phospho.app/).
 
-Please note the project is still under development.
+> This project is still under active development!
 
-## Requirements
+## Installation of the phospho client
 
-- Python 3.8 or higher
-
-## Installation
+You need Python `>=3.8`
 
 ```bash
 pip install --upgrade phospho
 ```
 
-## Getting started
+## Quickstart
+
+Create a phospho account and go to the [phospho dashboard](phospho.app). Create an API key and create a project. Set them as environment variables:
 
-Grab your API key from your phospho dashboard and set it as an environment variable:
 
 ```bash
+export PHOSPHO_PROJECT_ID="project-id"
 export PHOSPHO_API_KEY="your-api-key"
 ```
 
-Setup the project ID you want to work with (you can create a new project and get its ID from your dashboard):
+In your LLM app, log interactions with your agent using `phospho.log()`. Monitor and visualize your agent on the [phospho dashboard](phospho.app). 
 
-```bash
-export PHOSPHO_PROJECT_ID="your-project-id"
-```
+```python
+import phospho
+import openai
 
-Create a new client instance:
+phospho.init()
+openai_client = openai.OpenAI()
 
-```python
-from phospho import Client
+# This is your agent code
+query = {
+    "messages": [{"role": "user", "content": "Say this is a test"}], 
+    "model": "gpt-3.5-turbo", 
+}
+response = openai_client.chat.completions.create(**query)
 
-client = Client()
+# This is how you log it to phospho
+phospho.log(input=query, output=response)
 ```
 
+Read the docs at [docs.phospho.app](https://docs.phospho.app/) to go further. 
+
 ## Manage the sessions of your project
 
 Create a new session:

examples/hello.py

@@ -0,0 +1,17 @@
+import phospho
+import openai
+
+from pprint import pprint
+
+phospho.init()
+openai_client = openai.OpenAI()
+
+query = {
+    "messages": [{"role": "user", "content": "Say hi !"}],
+    "model": "gpt-3.5-turbo",
+}
+response = openai_client.chat.completions.create(**query)
+logged_content = phospho.log(input=query, output=response)
+
+print("The following content has been logged to phospho:")
+pprint(logged_content)

examples/metadata.py

@@ -0,0 +1,16 @@
+import phospho
+import openai
+
+phospho.init()
+
+# You can log more stuff than just input and output
+phospho.log(
+    input="input",
+    # Output logging is optional and can be set to None
+    output=None,
+    # Any other additional field will be logged
+    metadata={"some additonal metadata": "hello"},
+    any_other_name="some_value",
+    # Fields that aren't json serializable will be ignored and will raise a warning
+    invalid_field=lambda x: x,
+)

examples/session.py

@@ -0,0 +1,40 @@
+import phospho
+import openai
+
+from typing import Tuple, Dict, Any
+
+phospho.init(tick=0.5)
+openai_client = openai.OpenAI()
+
+
+def make_agent_do_stuff() -> Tuple[Dict[str, Any], Dict[str, Any]]:
+    query = {
+        "messages": [{"role": "user", "content": "Say hi !"}],
+        "model": "gpt-3.5-turbo",
+    }
+    response = openai_client.chat.completions.create(**query)
+
+    return query, response
+
+
+# By default, a new session id is created at the beginning
+input, output = make_agent_do_stuff()
+phospho_log = phospho.log(input=input, output=output)
+print(
+    f"No session_id were specified. Phospho created this one: {phospho_log['session_id']} (task_id: {phospho_log['task_id']})"
+)
+
+# And by default new logs keep this same session_id
+input, output = make_agent_do_stuff()
+phospho_log = phospho.log(input=input, output=output)
+print(
+    f"Just like before, the session_id is still: {phospho_log['session_id']} (task_id: {phospho_log['task_id']})"
+)
+
+# You can customize sessions by passing them as arguments to the logging function
+new_session_id = phospho.generate_uuid()
+input, output = make_agent_do_stuff()
+phospho_log = phospho.log(input=input, output=output, session_id=new_session_id)
+print(
+    f"The new session_id is: {phospho_log['session_id']} (task_id: {phospho_log['task_id']})"
+)

phospho/__init__.py

@@ -1,3 +1,112 @@
 from .agent import Agent
 from .message import Message
-from .client import Client
+from .client import Client
+from .consumer import Consumer
+from .log_queue import LogQueue
+from .utils import generate_timestamp, generate_uuid, convert_to_jsonable_dict
+from .extractor import get_input_output, RawDataType
+
+import logging
+
+from typing import Dict, Any, Optional, Union, Callable
+
+
+client = None
+log_queue = None
+consumer = None
+current_session_id = None
+verbose = True
+
+logger = logging.getLogger("phospho")
+
+
+def init(verbose: bool = True, tick: float = 0.5) -> None:
+    global client
+    global log_queue
+    global consumer
+    global current_session_id
+
+    client = Client()
+    log_queue = LogQueue()
+    consumer = Consumer(log_queue=log_queue, client=client, verbose=verbose, tick=tick)
+    # Start the consumer on a separate thread (this will periodically send logs to backend)
+    consumer.start()
+
+    # Initialize session and task id
+    if current_session_id is None:
+        current_session_id = generate_uuid()
+
+
+def log(
+    input: Union[RawDataType, str],
+    output: Optional[Union[RawDataType, str]] = None,
+    session_id: Optional[str] = None,
+    task_id: Optional[str] = None,
+    raw_input: Optional[RawDataType] = None,
+    raw_output: Optional[RawDataType] = None,
+    input_to_str_function: Optional[Callable[[Any], str]] = None,
+    output_to_str_function: Optional[Callable[[Any], str]] = None,
+    **kwargs: Dict[str, Any],
+) -> Dict[str, object]:
+    """Main logging endpoint
+
+    Returns: What has been logged
+    """
+    global client
+    global log_queue
+    global current_session_id
+    global verbose
+
+    assert (
+        (log_queue is not None) and (client is not None)
+    ), "phospho.log() was called but the global variable log_queue was not found. Make sure that phospho.init() was called."
+
+    # Session: if nothing specified, reuse existing id. Otherwise, update current id
+    if not session_id:
+        session_id = current_session_id
+    else:
+        current_session_id = session_id
+    # Task: if nothing specified, create new id.
+    if task_id is None:
+        task_id = generate_uuid()
+
+    # Process the input and output to convert them to dict
+    input_to_log, output_to_log, raw_input_to_log, raw_output_to_log = get_input_output(
+        input=input,
+        output=output,
+        raw_input=raw_input,
+        raw_output=raw_output,
+        input_to_str_function=input_to_str_function,
+        output_to_str_function=output_to_str_function,
+        verbose=verbose,
+    )
+
+    # Every other kwargs will be directly stored in the logs, if it's json serializable
+    if kwargs:
+        kwargs_to_log = convert_to_jsonable_dict(kwargs, verbose=verbose)
+    else:
+        kwargs_to_log = {}
+
+    # The log event looks like this:
+    log_event: Dict[str, object] = {
+        "client_created_at": generate_timestamp(),
+        # metadata
+        "project_id": client._project_id(),
+        "session_id": session_id,
+        "task_id": task_id,
+        # input
+        "input": str(input_to_log),
+        "raw_input": raw_input_to_log,
+        "raw_input_type_name": type(input).__name__,
+        # output
+        "output": str(output_to_log),
+        "raw_output": raw_output_to_log,
+        "raw_output_type_name": type(output).__name__,
+        # other
+        **kwargs_to_log,
+    }
+
+    # Append event to log_queue
+    log_queue.append(log_event)
+
+    return log_event