diff --git a/docs/source/apidocs/phospho/phospho.client.md b/docs/source/apidocs/phospho/phospho.client.md index 2afa43f1c..1efaab5e8 100644 --- a/docs/source/apidocs/phospho/phospho.client.md +++ b/docs/source/apidocs/phospho/phospho.client.md @@ -93,7 +93,7 @@ ```` -````{py:method} compare(context_input: str, old_output: str, new_output: str) -> phospho.evals.Comparison +````{py:method} compare(context_input: str, old_output: str, new_output: str, test_id: typing.Optional[str] = None) -> phospho.models.Comparison :canonical: phospho.client.Client.compare ```{autodoc2-docstring} phospho.client.Client.compare @@ -109,4 +109,20 @@ ```` +````{py:method} create_test(summary: typing.Optional[dict] = None) -> phospho.models.Test +:canonical: phospho.client.Client.create_test + +```{autodoc2-docstring} phospho.client.Client.create_test +``` + +```` + +````{py:method} update_test(test_id: str, status: typing.Literal[completed, canceled]) -> phospho.models.Test +:canonical: phospho.client.Client.update_test + +```{autodoc2-docstring} phospho.client.Client.update_test +``` + +```` + ````` diff --git a/docs/source/apidocs/phospho/phospho.evals.md b/docs/source/apidocs/phospho/phospho.evals.md deleted file mode 100644 index 61464cbbf..000000000 --- a/docs/source/apidocs/phospho/phospho.evals.md +++ /dev/null @@ -1,150 +0,0 @@ -# {py:mod}`phospho.evals` - -```{py:module} phospho.evals -``` - -```{autodoc2-docstring} phospho.evals -:allowtitles: -``` - -## Module Contents - -### Classes - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`Comparison ` - - -```` - -### Data - -````{list-table} -:class: autosummary longtable -:align: left - -* - {py:obj}`ComparisonResults ` - - ```{autodoc2-docstring} phospho.evals.ComparisonResults - :summary: - ``` -```` - -### API - -````{py:data} ComparisonResults -:canonical: phospho.evals.ComparisonResults -:value: > - None - -```{autodoc2-docstring} phospho.evals.ComparisonResults -``` - -```` - -`````{py:class} Comparison(**data: typing.Any) -:canonical: phospho.evals.Comparison - -Bases: {py:obj}`pydantic.BaseModel` - -````{py:attribute} id -:canonical: phospho.evals.Comparison.id -:type: str -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.id -``` - -```` - -````{py:attribute} created_at -:canonical: phospho.evals.Comparison.created_at -:type: int -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.created_at -``` - -```` - -````{py:attribute} project_id -:canonical: phospho.evals.Comparison.project_id -:type: str -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.project_id -``` - -```` - -````{py:attribute} instructions -:canonical: phospho.evals.Comparison.instructions -:type: typing.Optional[str] -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.instructions -``` - -```` - -````{py:attribute} context_input -:canonical: phospho.evals.Comparison.context_input -:type: str -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.context_input -``` - -```` - -````{py:attribute} old_output -:canonical: phospho.evals.Comparison.old_output -:type: str -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.old_output -``` - -```` - -````{py:attribute} new_output -:canonical: phospho.evals.Comparison.new_output -:type: str -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.new_output -``` - -```` - -````{py:attribute} comparison_result -:canonical: phospho.evals.Comparison.comparison_result -:type: phospho.evals.ComparisonResults -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.comparison_result -``` - -```` - -````{py:attribute} source -:canonical: phospho.evals.Comparison.source -:type: str -:value: > - None - -```{autodoc2-docstring} phospho.evals.Comparison.source -``` - -```` - -````` diff --git a/docs/source/apidocs/phospho/phospho.extractor.md b/docs/source/apidocs/phospho/phospho.extractor.md index 5b3049442..79cb9a9d1 100644 --- a/docs/source/apidocs/phospho/phospho.extractor.md +++ b/docs/source/apidocs/phospho/phospho.extractor.md @@ -92,7 +92,7 @@ ``` ```` -````{py:function} get_input_output(input: typing.Union[phospho.extractor.RawDataType, str], output: typing.Optional[typing.Union[phospho.extractor.RawDataType, str]] = None, raw_input: typing.Optional[phospho.extractor.RawDataType] = None, raw_output: typing.Optional[phospho.extractor.RawDataType] = None, input_to_str_function: typing.Optional[typing.Callable[[typing.Any], str]] = None, output_to_str_function: typing.Optional[typing.Callable[[typing.Any], str]] = None) -> typing.Tuple[str, typing.Optional[str], typing.Optional[typing.Union[typing.Dict[str, object], str]], typing.Optional[typing.Union[typing.Dict[str, object], str]], typing.Optional[str], typing.Optional[bool]] +````{py:function} get_input_output(input: typing.Union[phospho.extractor.RawDataType, str], output: typing.Optional[typing.Union[phospho.extractor.RawDataType, str]] = None, raw_input: typing.Optional[phospho.extractor.RawDataType] = None, raw_output: typing.Optional[phospho.extractor.RawDataType] = None, input_to_str_function: typing.Optional[typing.Callable[[typing.Any], str]] = None, output_to_str_function: typing.Optional[typing.Callable[[typing.Any], str]] = None) -> typing.Tuple[str, typing.Optional[str], typing.Optional[typing.Union[typing.Dict[str, object], str]], typing.Optional[typing.Union[typing.Dict[str, object], str]]] :canonical: phospho.extractor.get_input_output ```{autodoc2-docstring} phospho.extractor.get_input_output diff --git a/docs/source/apidocs/phospho/phospho.integrations.md b/docs/source/apidocs/phospho/phospho.integrations.md new file mode 100644 index 000000000..3b3415f17 --- /dev/null +++ b/docs/source/apidocs/phospho/phospho.integrations.md @@ -0,0 +1,8 @@ +# {py:mod}`phospho.integrations` + +```{py:module} phospho.integrations +``` + +```{autodoc2-docstring} phospho.integrations +:allowtitles: +``` diff --git a/docs/source/apidocs/phospho/phospho.md b/docs/source/apidocs/phospho/phospho.md index d1a20042d..92d0dc8b4 100644 --- a/docs/source/apidocs/phospho/phospho.md +++ b/docs/source/apidocs/phospho/phospho.md @@ -18,10 +18,11 @@ phospho.tasks phospho.config phospho.models phospho.client -phospho.evals phospho.extractor phospho.utils phospho.collection +phospho.testing +phospho.integrations ``` ## Package Contents @@ -107,7 +108,7 @@ phospho.collection ````{py:data} __all__ :canonical: phospho.__all__ :value: > - ['Client', 'Consumer', 'LogQueue', 'Event', 'generate_timestamp', 'generate_uuid', 'convert_to_jsona... + ['Client', 'Consumer', 'LogQueue', 'Event', 'generate_timestamp', 'generate_uuid', 'filter_nonjsonab... ```{autodoc2-docstring} phospho.__all__ ``` @@ -199,7 +200,7 @@ phospho.collection ``` ```` -````{py:function} log(input: typing.Union[phospho.extractor.RawDataType, str], output: typing.Optional[typing.Union[phospho.extractor.RawDataType, str, typing.Iterable[phospho.extractor.RawDataType]]] = None, session_id: typing.Optional[str] = None, task_id: typing.Optional[str] = None, raw_input: typing.Optional[phospho.extractor.RawDataType] = None, raw_output: typing.Optional[phospho.extractor.RawDataType] = None, input_to_str_function: typing.Optional[typing.Callable[[typing.Any], str]] = None, output_to_str_function: typing.Optional[typing.Callable[[typing.Any], str]] = None, output_to_task_id_and_to_log_function: typing.Optional[typing.Callable[[typing.Any], typing.Tuple[typing.Optional[str], bool]]] = None, concatenate_raw_outputs_if_task_id_exists: bool = True, stream: bool = False, **kwargs: typing.Dict[str, typing.Any]) -> typing.Dict[str, object] +````{py:function} log(input: typing.Union[phospho.extractor.RawDataType, str], output: typing.Optional[typing.Union[phospho.extractor.RawDataType, str, typing.Iterable[phospho.extractor.RawDataType]]] = None, session_id: typing.Optional[str] = None, task_id: typing.Optional[str] = None, raw_input: typing.Optional[phospho.extractor.RawDataType] = None, raw_output: typing.Optional[phospho.extractor.RawDataType] = None, input_to_str_function: typing.Optional[typing.Callable[[typing.Any], str]] = None, output_to_str_function: typing.Optional[typing.Callable[[typing.Any], str]] = None, concatenate_raw_outputs_if_task_id_exists: bool = True, stream: bool = False, **kwargs: typing.Dict[str, typing.Any]) -> typing.Optional[typing.Dict[str, object]] :canonical: phospho.log ```{autodoc2-docstring} phospho.log diff --git a/docs/source/apidocs/phospho/phospho.models.md b/docs/source/apidocs/phospho/phospho.models.md index b2f8ff2aa..e0f60998b 100644 --- a/docs/source/apidocs/phospho/phospho.models.md +++ b/docs/source/apidocs/phospho/phospho.models.md @@ -19,6 +19,22 @@ - * - {py:obj}`TaskModel ` - +* - {py:obj}`Test ` + - +* - {py:obj}`Comparison ` + - +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`ComparisonResults ` + - ```{autodoc2-docstring} phospho.models.ComparisonResults + :summary: + ``` ```` ### API @@ -267,3 +283,214 @@ Bases: {py:obj}`pydantic.BaseModel` ```` ````` + +`````{py:class} Test(**data: typing.Any) +:canonical: phospho.models.Test + +Bases: {py:obj}`pydantic.BaseModel` + +````{py:attribute} id +:canonical: phospho.models.Test.id +:type: str +:value: > + 'Field(...)' + +```{autodoc2-docstring} phospho.models.Test.id +``` + +```` + +````{py:attribute} project_id +:canonical: phospho.models.Test.project_id +:type: str +:value: > + None + +```{autodoc2-docstring} phospho.models.Test.project_id +``` + +```` + +````{py:attribute} created_by +:canonical: phospho.models.Test.created_by +:type: str +:value: > + None + +```{autodoc2-docstring} phospho.models.Test.created_by +``` + +```` + +````{py:attribute} created_at +:canonical: phospho.models.Test.created_at +:type: int +:value: > + 'Field(...)' + +```{autodoc2-docstring} phospho.models.Test.created_at +``` + +```` + +````{py:attribute} last_updated_at +:canonical: phospho.models.Test.last_updated_at +:type: int +:value: > + None + +```{autodoc2-docstring} phospho.models.Test.last_updated_at +``` + +```` + +````{py:attribute} terminated_at +:canonical: phospho.models.Test.terminated_at +:type: typing.Optional[int] +:value: > + None + +```{autodoc2-docstring} phospho.models.Test.terminated_at +``` + +```` + +````{py:attribute} status +:canonical: phospho.models.Test.status +:type: typing.Literal[started, completed, canceled] +:value: > + None + +```{autodoc2-docstring} phospho.models.Test.status +``` + +```` + +````{py:attribute} summary +:canonical: phospho.models.Test.summary +:type: dict +:value: > + 'Field(...)' + +```{autodoc2-docstring} phospho.models.Test.summary +``` + +```` + +````` + +````{py:data} ComparisonResults +:canonical: phospho.models.ComparisonResults +:value: > + None + +```{autodoc2-docstring} phospho.models.ComparisonResults +``` + +```` + +`````{py:class} Comparison(**data: typing.Any) +:canonical: phospho.models.Comparison + +Bases: {py:obj}`pydantic.BaseModel` + +````{py:attribute} id +:canonical: phospho.models.Comparison.id +:type: str +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.id +``` + +```` + +````{py:attribute} created_at +:canonical: phospho.models.Comparison.created_at +:type: int +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.created_at +``` + +```` + +````{py:attribute} project_id +:canonical: phospho.models.Comparison.project_id +:type: str +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.project_id +``` + +```` + +````{py:attribute} instructions +:canonical: phospho.models.Comparison.instructions +:type: typing.Optional[str] +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.instructions +``` + +```` + +````{py:attribute} context_input +:canonical: phospho.models.Comparison.context_input +:type: str +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.context_input +``` + +```` + +````{py:attribute} old_output +:canonical: phospho.models.Comparison.old_output +:type: str +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.old_output +``` + +```` + +````{py:attribute} new_output +:canonical: phospho.models.Comparison.new_output +:type: str +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.new_output +``` + +```` + +````{py:attribute} comparison_result +:canonical: phospho.models.Comparison.comparison_result +:type: phospho.models.ComparisonResults +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.comparison_result +``` + +```` + +````{py:attribute} source +:canonical: phospho.models.Comparison.source +:type: str +:value: > + None + +```{autodoc2-docstring} phospho.models.Comparison.source +``` + +```` + +````` diff --git a/docs/source/apidocs/phospho/phospho.tasks.md b/docs/source/apidocs/phospho/phospho.tasks.md index 9c8416d2d..bcac90a0b 100644 --- a/docs/source/apidocs/phospho/phospho.tasks.md +++ b/docs/source/apidocs/phospho/phospho.tasks.md @@ -53,6 +53,14 @@ ```` +````{py:method} content_as_dict() -> dict +:canonical: phospho.tasks.Task.content_as_dict + +```{autodoc2-docstring} phospho.tasks.Task.content_as_dict +``` + +```` + ````{py:method} refresh() -> None :canonical: phospho.tasks.Task.refresh diff --git a/docs/source/apidocs/phospho/phospho.testing.md b/docs/source/apidocs/phospho/phospho.testing.md new file mode 100644 index 000000000..4720baca6 --- /dev/null +++ b/docs/source/apidocs/phospho/phospho.testing.md @@ -0,0 +1,331 @@ +# {py:mod}`phospho.testing` + +```{py:module} phospho.testing +``` + +```{autodoc2-docstring} phospho.testing +:allowtitles: +``` + +## Module Contents + +### Classes + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`TestInput ` + - +* - {py:obj}`BacktestLoader ` + - ```{autodoc2-docstring} phospho.testing.BacktestLoader + :summary: + ``` +* - {py:obj}`DatasetLoader ` + - ```{autodoc2-docstring} phospho.testing.DatasetLoader + :summary: + ``` +* - {py:obj}`PhosphoTest ` + - ```{autodoc2-docstring} phospho.testing.PhosphoTest + :summary: + ``` +```` + +### Functions + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`adapt_dict_to_agent_function ` + - ```{autodoc2-docstring} phospho.testing.adapt_dict_to_agent_function + :summary: + ``` +* - {py:obj}`adapt_task_to_agent_function ` + - ```{autodoc2-docstring} phospho.testing.adapt_task_to_agent_function + :summary: + ``` +* - {py:obj}`adapt_to_sample_size ` + - ```{autodoc2-docstring} phospho.testing.adapt_to_sample_size + :summary: + ``` +```` + +### Data + +````{list-table} +:class: autosummary longtable +:align: left + +* - {py:obj}`logger ` + - ```{autodoc2-docstring} phospho.testing.logger + :summary: + ``` +```` + +### API + +````{py:data} logger +:canonical: phospho.testing.logger +:value: > + 'getLogger(...)' + +```{autodoc2-docstring} phospho.testing.logger +``` + +```` + +`````{py:class} TestInput(**data: typing.Any) +:canonical: phospho.testing.TestInput + +Bases: {py:obj}`pydantic.BaseModel` + +````{py:attribute} function_input +:canonical: phospho.testing.TestInput.function_input +:type: dict +:value: > + None + +```{autodoc2-docstring} phospho.testing.TestInput.function_input +``` + +```` + +````{py:attribute} input +:canonical: phospho.testing.TestInput.input +:type: typing.Optional[str] +:value: > + None + +```{autodoc2-docstring} phospho.testing.TestInput.input +``` + +```` + +````{py:attribute} additional_input +:canonical: phospho.testing.TestInput.additional_input +:type: typing.Optional[dict] +:value: > + None + +```{autodoc2-docstring} phospho.testing.TestInput.additional_input +``` + +```` + +````{py:attribute} output +:canonical: phospho.testing.TestInput.output +:type: typing.Optional[str] +:value: > + None + +```{autodoc2-docstring} phospho.testing.TestInput.output +``` + +```` + +````{py:attribute} task_id +:canonical: phospho.testing.TestInput.task_id +:type: typing.Optional[str] +:value: > + None + +```{autodoc2-docstring} phospho.testing.TestInput.task_id +``` + +```` + +````{py:method} from_task(task: phospho.tasks.Task) -> phospho.testing.TestInput +:canonical: phospho.testing.TestInput.from_task +:classmethod: + +```{autodoc2-docstring} phospho.testing.TestInput.from_task +``` + +```` + +````` + +````{py:function} adapt_dict_to_agent_function(dict_to_adapt: dict, agent_function: typing.Callable[[typing.Any], typing.Any]) -> typing.Optional[dict] +:canonical: phospho.testing.adapt_dict_to_agent_function + +```{autodoc2-docstring} phospho.testing.adapt_dict_to_agent_function +``` +```` + +````{py:function} adapt_task_to_agent_function(task: phospho.tasks.Task, agent_function: typing.Callable[[typing.Any], typing.Any]) -> typing.Optional[phospho.testing.TestInput] +:canonical: phospho.testing.adapt_task_to_agent_function + +```{autodoc2-docstring} phospho.testing.adapt_task_to_agent_function +``` +```` + +````{py:function} adapt_to_sample_size(list_to_sample, sample_size) +:canonical: phospho.testing.adapt_to_sample_size + +```{autodoc2-docstring} phospho.testing.adapt_to_sample_size +``` +```` + +`````{py:class} BacktestLoader(client: phospho.client.Client, agent_function: typing.Callable[[typing.Any], typing.Any], sample_size: typing.Optional[int] = 10) +:canonical: phospho.testing.BacktestLoader + +```{autodoc2-docstring} phospho.testing.BacktestLoader +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} phospho.testing.BacktestLoader.__init__ +``` + +````{py:method} __iter__() +:canonical: phospho.testing.BacktestLoader.__iter__ + +```{autodoc2-docstring} phospho.testing.BacktestLoader.__iter__ +``` + +```` + +````{py:method} __next__() +:canonical: phospho.testing.BacktestLoader.__next__ + +```{autodoc2-docstring} phospho.testing.BacktestLoader.__next__ +``` + +```` + +````{py:method} __len__() +:canonical: phospho.testing.BacktestLoader.__len__ + +```{autodoc2-docstring} phospho.testing.BacktestLoader.__len__ +``` + +```` + +````` + +`````{py:class} DatasetLoader(agent_function: typing.Callable[[typing.Any], typing.Any], path: str, test_n_times: typing.Optional[int] = None) +:canonical: phospho.testing.DatasetLoader + +```{autodoc2-docstring} phospho.testing.DatasetLoader +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} phospho.testing.DatasetLoader.__init__ +``` + +````{py:method} __iter__() +:canonical: phospho.testing.DatasetLoader.__iter__ + +```{autodoc2-docstring} phospho.testing.DatasetLoader.__iter__ +``` + +```` + +````{py:method} __next__() +:canonical: phospho.testing.DatasetLoader.__next__ + +```{autodoc2-docstring} phospho.testing.DatasetLoader.__next__ +``` + +```` + +````{py:method} __len__() +:canonical: phospho.testing.DatasetLoader.__len__ + +```{autodoc2-docstring} phospho.testing.DatasetLoader.__len__ +``` + +```` + +````` + +`````{py:class} PhosphoTest(api_key: typing.Optional[str] = None, project_id: typing.Optional[str] = None) +:canonical: phospho.testing.PhosphoTest + +```{autodoc2-docstring} phospho.testing.PhosphoTest +``` + +```{rubric} Initialization +``` + +```{autodoc2-docstring} phospho.testing.PhosphoTest.__init__ +``` + +````{py:attribute} client +:canonical: phospho.testing.PhosphoTest.client +:type: phospho.client.Client +:value: > + None + +```{autodoc2-docstring} phospho.testing.PhosphoTest.client +``` + +```` + +````{py:attribute} functions_to_evaluate +:canonical: phospho.testing.PhosphoTest.functions_to_evaluate +:type: typing.Dict[str, typing.Any] +:value: > + None + +```{autodoc2-docstring} phospho.testing.PhosphoTest.functions_to_evaluate +``` + +```` + +````{py:attribute} test_id +:canonical: phospho.testing.PhosphoTest.test_id +:type: typing.Optional[str] +:value: > + None + +```{autodoc2-docstring} phospho.testing.PhosphoTest.test_id +``` + +```` + +````{py:method} test(fn: typing.Optional[typing.Callable[[typing.Any], typing.Any]] = None, source_loader: typing.Literal[backtest, dataset] = 'backtest', source_loader_params: typing.Optional[typing.Dict[str, typing.Any]] = None, metrics: typing.Optional[typing.List[typing.Literal[compare, evaluate]]] = None) -> typing.Callable[[typing.Any], typing.Any] +:canonical: phospho.testing.PhosphoTest.test + +```{autodoc2-docstring} phospho.testing.PhosphoTest.test +``` + +```` + +````{py:method} get_output_from_agent(function_input: typing.Dict[str, typing.Any], agent_function: typing.Callable[[typing.Any], typing.Any], metric_name: str) +:canonical: phospho.testing.PhosphoTest.get_output_from_agent + +```{autodoc2-docstring} phospho.testing.PhosphoTest.get_output_from_agent +``` + +```` + +````{py:method} evaluate(task_to_evaluate: typing.Dict[str, typing.Any]) +:canonical: phospho.testing.PhosphoTest.evaluate + +```{autodoc2-docstring} phospho.testing.PhosphoTest.evaluate +``` + +```` + +````{py:method} compare(task_to_compare: typing.Dict[str, typing.Any]) -> None +:canonical: phospho.testing.PhosphoTest.compare + +```{autodoc2-docstring} phospho.testing.PhosphoTest.compare +``` + +```` + +````{py:method} run(executor_type: typing.Literal[parallel, sequential] = 'parallel') +:canonical: phospho.testing.PhosphoTest.run + +```{autodoc2-docstring} phospho.testing.PhosphoTest.run +``` + +```` + +````` diff --git a/docs/source/apidocs/phospho/phospho.utils.md b/docs/source/apidocs/phospho/phospho.utils.md index 28cb68b33..83d5a376d 100644 --- a/docs/source/apidocs/phospho/phospho.utils.md +++ b/docs/source/apidocs/phospho/phospho.utils.md @@ -47,8 +47,8 @@ - ```{autodoc2-docstring} phospho.utils.filter_nonjsonable_keys :summary: ``` -* - {py:obj}`convert_to_jsonable_dict ` - - ```{autodoc2-docstring} phospho.utils.convert_to_jsonable_dict +* - {py:obj}`convert_content_to_loggable_content ` + - ```{autodoc2-docstring} phospho.utils.convert_content_to_loggable_content :summary: ``` ```` @@ -98,17 +98,17 @@ ``` ```` -````{py:function} filter_nonjsonable_keys(arg_dict: dict) -> dict +````{py:function} filter_nonjsonable_keys(arg_dict: dict, verbose: bool = False) -> typing.Dict[str, object] :canonical: phospho.utils.filter_nonjsonable_keys ```{autodoc2-docstring} phospho.utils.filter_nonjsonable_keys ``` ```` -````{py:function} convert_to_jsonable_dict(arg_dict: dict, verbose: bool = False) -> typing.Dict[str, object] -:canonical: phospho.utils.convert_to_jsonable_dict +````{py:function} convert_content_to_loggable_content(content: typing.Any) -> typing.Union[typing.Dict[str, object], str] +:canonical: phospho.utils.convert_content_to_loggable_content -```{autodoc2-docstring} phospho.utils.convert_to_jsonable_dict +```{autodoc2-docstring} phospho.utils.convert_content_to_loggable_content ``` ```` diff --git a/phospho/__init__.py b/phospho/__init__.py index 3b26836d7..a389c9bca 100644 --- a/phospho/__init__.py +++ b/phospho/__init__.py @@ -46,7 +46,6 @@ "integrations", ] -import pydantic import logging from copy import deepcopy @@ -57,7 +56,6 @@ Optional, Union, Callable, - Tuple, Iterable, AsyncIterable, Coroutine, @@ -176,8 +174,6 @@ def _log_single_event( output_to_log, raw_input_to_log, raw_output_to_log, - task_id_from_output, - extracted_to_log, ) = get_input_output( input=input, output=output, @@ -187,17 +183,9 @@ def _log_single_event( output_to_str_function=output_to_str_function, ) - # Override to_log parameter - if extracted_to_log is not None: - to_log = extracted_to_log - # Task: use the task_id parameter, the task_id infered from inputs, or generate one if task_id is None: - if task_id_from_output is None: - # if nothing specified, create new id. - task_id = generate_uuid() - else: - task_id = task_id_from_output + task_id = generate_uuid() # Keep track of the latest task_id and session_id latest_task_id = task_id @@ -410,9 +398,6 @@ def log( # todo: group those into "transformation" input_to_str_function: Optional[Callable[[Any], str]] = None, output_to_str_function: Optional[Callable[[Any], str]] = None, - output_to_task_id_and_to_log_function: Optional[ - Callable[[Any], Tuple[Optional[str], bool]] - ] = None, concatenate_raw_outputs_if_task_id_exists: bool = True, stream: bool = False, **kwargs: Dict[str, Any], @@ -432,12 +417,12 @@ def log( `session_id` is used to group logs together. For example, a single conversation. - By default, every log is assigned to a unique `task_id` and is immediately pushed to backend. - However, if you pass multiple logs with the same `task_id` and `to_log=False`, they will - stay in queue until they receive the same `task_id` with `to_log=False`. They will then - be combined and pushed to backend. - You can automate this behaviour using `output_to_task_id_and_to_log_function`. This is used - to handle streaming. + `task_id` is used to identify a single task. For example, a single message in a conversation. + This is useful to log user feedback on a specific task (see phospho.user_feedback). + + `stream` is used to log a stream of data. For example, a generator. If `stream=True`, then + `phospho.log` returns a generator that also logs every individual output. See `phospho.wrap` + for more details. Every other `**kwargs` will be added to the log content and stored. @@ -484,7 +469,6 @@ def log( "raw_output": raw_output, "input_to_str_function": input_to_str_function, "output_to_str_function": output_to_str_function, - "output_to_task_id_and_to_log_function": output_to_task_id_and_to_log_function, "concatenate_raw_outputs_if_task_id_exists": concatenate_raw_outputs_if_task_id_exists, } # Return the log: @@ -514,7 +498,6 @@ def log( raw_output=raw_output, input_to_str_function=input_to_str_function, output_to_str_function=output_to_str_function, - output_to_task_id_and_to_log_function=output_to_task_id_and_to_log_function, concatenate_raw_outputs_if_task_id_exists=concatenate_raw_outputs_if_task_id_exists, to_log=True, **kwargs, @@ -764,7 +747,8 @@ def user_feedback( Note: Feedback can be directly logged with `phospho.log` by passing `flag` as a keyword argument. - :param task_id: The task_id of the task to flag + :param task_id: The task_id of the task to flag. Get the task_id from the returned value of phospho.log, use phospho.new_task + to generate a new task_id, or use pospho.latest_task_id. :param flag: The flag to set for the task. Either "success" or "failure" :param notes: Optional notes to add to the task. For example, the reason for the flag. :param source: The source of the feedback, such as "user", "system", "user@mail.com", etc. diff --git a/phospho/config.py b/phospho/config.py index 17792f83b..549d8d697 100644 --- a/phospho/config.py +++ b/phospho/config.py @@ -2,4 +2,4 @@ Phospho internal configuration file """ -BASE_URL = "https://api.phospho.ai/v0" +BASE_URL = "https://api.phospho.ai/v2" diff --git a/phospho/extractor.py b/phospho/extractor.py index ea85efd90..ab6292426 100644 --- a/phospho/extractor.py +++ b/phospho/extractor.py @@ -170,8 +170,6 @@ def get_input_output( Optional[str], Optional[Union[Dict[str, object], str]], Optional[Union[Dict[str, object], str]], - Optional[str], - Optional[bool], ]: """ Convert any supported data type to standard, loggable inputs and outputs. @@ -192,8 +190,6 @@ def get_input_output( :param output_to_str_function: - :param verbose: - :return: - input_to_log _(str)_ - @@ -208,11 +204,6 @@ def get_input_output( - raw_output_to_log _(Optional[Dict[str, object]])_ - A dict representation of the output, raw_output if specified, or None if output is a str. - - task_id_from_output _(Optional[str])_ - - Task id detected from the output. Useful from keeping track of streaming outputs. - - - to_log _(Optional[bool])_ - - Whether to log the event directly, or wait until a later event. Useful for streaming. """ # Default functions to extract string from input and output @@ -225,9 +216,6 @@ def get_input_output( raw_input_to_log: Optional[Union[Dict[str, object], str]] = None raw_output_to_log: Optional[Union[Dict[str, object], str]] = None - task_id_from_output = None - to_log = None - # Extract a string representation from input if isinstance(input, str): input_to_log = input @@ -248,7 +236,6 @@ def get_input_output( raw_output_to_log = output else: output_to_log = output_to_str_function(output) - # task_id_from_output, to_log = output_to_task_id_and_to_log_function(output) raw_output_to_log = filter_nonjsonable_keys(convert_to_dict(output)) else: output_to_log = None @@ -262,6 +249,4 @@ def get_input_output( output_to_log, raw_input_to_log, raw_output_to_log, - task_id_from_output, - to_log, ) diff --git a/pyproject.toml b/pyproject.toml index b236490d5..cfb4c2952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "phospho" -version = "0.2.8" +version = "0.2.9" description = "Analytics for LLM apps" authors = ["Pierre-Louis Biojout "] readme = "README.md"