diff --git a/CHANGELOG.md b/CHANGELOG.md index 01c0b0c..0742908 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.3.3] - In progress... +### Added +- [#63](https://github.com/brunohjs/rasa-model-report/issues/63) Created `--precision` CLI command parameter. This command is used to change precision of the model report overview grades. + ## [1.3.2] - 2023-02-26 ### Added - [#26](https://github.com/brunohjs/rasa-model-report/issues/26) Updated release script. @@ -70,6 +74,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [#16](https://github.com/brunohjs/rasa-model-report/issues/16) Created a handler for retrieval intents in the report. +[1.3.3]: https://github.com/brunohjs/rasa-model-report/compare/1.3.2...HEAD [1.3.2]: https://github.com/brunohjs/rasa-model-report/compare/1.3.0...1.3.2 [1.3.1]: https://github.com/brunohjs/rasa-model-report/compare/1.3.0...1.3.1 [1.3.0]: https://github.com/brunohjs/rasa-model-report/compare/1.2.0...1.3.0 diff --git a/README.md b/README.md index adc333d..6f68860 100644 --- a/README.md +++ b/README.md @@ -93,14 +93,15 @@ There are parameters that can be used. Available options are below: path) --disable-nlu Disable processing NLU sentences. NLU section will not be generated in the report. Required Rasa API. - (default: false) -h, --help Show this help message. --model-link TEXT Model download link. It's only displayed in the report to model download. ---no-images Generate model report without images. (default: - false) +--no-images Generate model report without images. --output-path TEXT Report output path. (default: ./) -p, --path TEXT Rasa project path. (default: ./) +--precision INTEGER Grade precision. Used to change precision of the + model report overview grades. Can vary between 0 and + 5 (default: 1) --project-name TEXT Rasa project name. It's only displayed in the report. (default: My project) --project-version TEXT Project version. It's only displayed in the report diff --git a/rasa_model_report/controllers/json_controller.py b/rasa_model_report/controllers/json_controller.py index 9b05208..881f818 100644 --- a/rasa_model_report/controllers/json_controller.py +++ b/rasa_model_report/controllers/json_controller.py @@ -216,6 +216,10 @@ def save_overview(self) -> None: file.write("\n") file.close() + @staticmethod + def weight_function(x: float) -> float: + return 1 - x**2 + def _calculate_overall(self) -> None: """ Calculate report overall. @@ -224,15 +228,15 @@ def _calculate_overall(self) -> None: "intent": 2, "entity": 1, "core": 1, - "nlu": 3, - "e2e_coverage": 3 + "nlu": 8, + "e2e_coverage": 8 } overview_sum = sum( - self._overview[item] * w for item, w in weights.items() + self._overview[item] * self.weight_function(self._overview[item]) for item in weights if isinstance(self._overview[item], (int, float)) and self._overview[item] >= 0 ) weight_sum = sum( - w for item, w in weights.items() + self.weight_function(self._overview[item]) for item in weights if isinstance(self._overview[item], (int, float)) and self._overview[item] >= 0 ) overview_rate = overview_sum / weight_sum if weight_sum else 0 diff --git a/rasa_model_report/controllers/markdown_controller.py b/rasa_model_report/controllers/markdown_controller.py index fcb5d3b..b3f2cd3 100644 --- a/rasa_model_report/controllers/markdown_controller.py +++ b/rasa_model_report/controllers/markdown_controller.py @@ -10,6 +10,7 @@ from rasa_model_report.controllers.e2e_coverage_controller import E2ECoverageController from rasa_model_report.controllers.json_controller import JsonController from rasa_model_report.controllers.nlu_controller import NluController +from rasa_model_report.helpers import constants from rasa_model_report.helpers import type_aliases from rasa_model_report.helpers import utils @@ -44,7 +45,8 @@ def __init__( self.output_report_path: str = utils.remove_duplicate_slashs(f"{self.output_path}/model_report.md") self.readme_path: str = "README.md" self.model_link: str = kwargs.get("model_link") - self.no_images: bool = kwargs.get("no_images", False) + self.no_images: bool = kwargs.get("no_images", constants.NO_IMAGES) + self.precision: int = kwargs.get("precision", constants.GRADE_PRECISION) self.json: JsonController = JsonController(rasa_path, output_path, project_name, project_version) self.csv: CsvController = CsvController(rasa_path, output_path, project_name, project_version) self.nlu: NluController = NluController( @@ -52,8 +54,8 @@ def __init__( output_path, project_name, project_version, - url=kwargs.get("rasa_api_url"), - disable_nlu=kwargs.get("disable_nlu") + url=kwargs.get("rasa_api_url", constants.RASA_API_URL), + disable_nlu=kwargs.get("disable_nlu", constants.DISABLE_NLU) ) self.e2e_coverage: E2ECoverageController = E2ECoverageController( rasa_path, @@ -182,12 +184,12 @@ def build_overview(self) -> str: style = "style='font-size:20px'" text += f"|Intent|Entity|NLU|Core|E2E Coverage|General|\n" text += "|:-:|:-:|:-:|:-:|:-:|:-:|\n" - text += f"|{utils.change_scale(overview['intent'], 10)}\ - |{utils.change_scale(overview['entity'], 10)}\ - |{utils.change_scale(overview['nlu'], 10)}\ - |{utils.change_scale(overview['core'], 10)}\ - |{utils.change_scale(overview['e2e_coverage'], 10)}\ - |**{utils.change_scale(overview['overall'], 10)}**|\n" + text += f"|{utils.change_scale(overview['intent'], 10, self.precision)}\ + |{utils.change_scale(overview['entity'], 10, self.precision)}\ + |{utils.change_scale(overview['nlu'], 10, self.precision)}\ + |{utils.change_scale(overview['core'], 10, self.precision)}\ + |{utils.change_scale(overview['e2e_coverage'], 10, self.precision)}\ + |**{utils.change_scale(overview['overall'], 10, self.precision)}**|\n" text += f"{utils.get_color(overview['intent'])}\ |{utils.get_color(overview['entity'])}\ |{utils.get_color(overview['nlu'])}\ diff --git a/rasa_model_report/controllers/nlu_controller.py b/rasa_model_report/controllers/nlu_controller.py index 95b8b34..8cb3435 100644 --- a/rasa_model_report/controllers/nlu_controller.py +++ b/rasa_model_report/controllers/nlu_controller.py @@ -10,6 +10,7 @@ import requests.exceptions from rasa_model_report.controllers.controller import Controller +from rasa_model_report.helpers import constants from rasa_model_report.helpers import type_aliases from rasa_model_report.helpers import utils @@ -41,7 +42,7 @@ def __init__( self._problem_sentences: List[type_aliases.nlu_payload] = [] self._general_grade: Optional[float] = None self._connected: bool = False - self._disable_nlu: bool = kwargs.get("disable_nlu") + self._disable_nlu: bool = kwargs.get("disable_nlu", constants.DISABLE_NLU) self.url: str = url if not self._disable_nlu and self.health_check_rasa_api(): @@ -108,7 +109,7 @@ def _generate_data(self) -> List[type_aliases.nlu_payload]: for intent, examples in self._data.items(): progress = index / len(self._data) * 100 index += 1 - logging.info(f" - Analyzing NLU of the {intent} intent ({progress:<5.1f}%).") + logging.info(f" - ({progress:<5.1f}%) analyzing NLU intent: {intent}") for text in examples: text = self.remove_entities_from_text(text) nlu_requested = self.request_nlu(text) diff --git a/rasa_model_report/helpers/constants.py b/rasa_model_report/helpers/constants.py new file mode 100644 index 0000000..7cccd7a --- /dev/null +++ b/rasa_model_report/helpers/constants.py @@ -0,0 +1,9 @@ +DISABLE_NLU = False +GRADE_PRECISION = 1 +NO_IMAGES = False +OUTPUT_PATH = "./" +PROJECT_NAME = "My project" +PROJECT_VERSION = None +RASA_API_URL = "http://localhost:5005" +RASA_PATH = "./" +RASA_VERSION = None diff --git a/rasa_model_report/helpers/type_aliases.py b/rasa_model_report/helpers/type_aliases.py index a61db1f..b52b339 100644 --- a/rasa_model_report/helpers/type_aliases.py +++ b/rasa_model_report/helpers/type_aliases.py @@ -2,6 +2,7 @@ from typing import List from typing import Union + intent = Dict[str, Union[float, str, None]] entity = Dict[str, Union[float, Dict[str, float], None]] nlu_payload = Dict[str, Union[intent, List[intent], str, float, None]] diff --git a/rasa_model_report/helpers/utils.py b/rasa_model_report/helpers/utils.py index 4aca15d..82ab37e 100644 --- a/rasa_model_report/helpers/utils.py +++ b/rasa_model_report/helpers/utils.py @@ -11,6 +11,8 @@ from requests.adapters import Retry from yaml import safe_load +from rasa_model_report.helpers import constants + def format_date() -> str: """ @@ -64,16 +66,28 @@ def get_color(value: float, scale: int = 1) -> str: return "❌" -def change_scale(value: float, scale: int = 1) -> str: +def change_scale(value: float, scale: int = 1, precision: int = 1) -> str: """ Change the value scale and rounds it to display in string format. :param value: Value that will be changed to scale and rounds it. :param scale: Scale that will be applied. - :return: Value on the new scale. - """ - if isinstance(value, (float, int)): - return f"{int(value * scale)}" + :param precision: Value precision. + :return: Value on the new scale and precision. + """ + precision = precision if isinstance(precision, int) and 5 >= precision >= 0 else constants.GRADE_PRECISION + if ( + isinstance(value, (float, int)) and + isinstance(scale, (float, int)) and + scale != 0 + ): + new_value = value * scale + if new_value >= 1 and new_value % int(new_value) == 0: + return str(int(new_value)) + elif re.search(r"\.0$", f"{new_value:.{precision}f}"): + return "0" + else: + return f"{new_value:.{precision}f}" return value diff --git a/rasa_model_report/main.py b/rasa_model_report/main.py index f67a2fe..45312bb 100644 --- a/rasa_model_report/main.py +++ b/rasa_model_report/main.py @@ -3,6 +3,7 @@ import click from rasa_model_report.controllers.model_report import ModelReport +from rasa_model_report.helpers import constants logging.basicConfig(format="%(asctime)s [%(levelname)s] %(message)s", level=logging.INFO) @@ -17,9 +18,9 @@ "--disable-nlu", is_flag=True, required=False, - default=False, + default=constants.DISABLE_NLU, help="Disable processing NLU sentences. NLU section will not be generated " - "in the report. Required Rasa API. (default: false)" + "in the report. Required Rasa API." ) @click.help_option( "--help", @@ -36,50 +37,58 @@ "--no-images", is_flag=True, required=False, - default=False, - help="Generate model report without images. (default: false)" + default=constants.NO_IMAGES, + help="Generate model report without images." ) @click.option( "--output-path", type=str, required=False, - default="./", - help="Report output path. (default: ./)" + default=constants.OUTPUT_PATH, + help=f"Report output path. (default: {constants.OUTPUT_PATH})" ) @click.option( "--path", "-p", type=str, required=False, - default="./", - help="Rasa project path. (default: ./)" + default=constants.RASA_PATH, + help=f"Rasa project path. (default: {constants.RASA_PATH})" +) +@click.option( + "--precision", + type=int, + required=False, + default=constants.GRADE_PRECISION, + help="Grade precision. Used to change precision of the model report overview grades. " + f"Can vary between 0 and 5 (default: {constants.GRADE_PRECISION})" ) @click.option( "--project-name", type=str, required=False, - default="My Project", - help="Rasa project name. It's only displayed in the report. (default: My project)" + default=constants.PROJECT_NAME, + help=f"Rasa project name. It's only displayed in the report. (default: {constants.PROJECT_NAME})" ) @click.option( "--project-version", type=str, required=False, - default=None, + default=constants.PROJECT_VERSION, help="Project version. It's only displayed in the report for project documentation." ) @click.option( "--rasa-api", type=str, required=False, - default="http://localhost:5005", - help="Rasa API URL. Is needed to create NLU section of report. (default: http://localhost:5005)" + default=constants.RASA_API_URL, + help=f"Rasa API URL. Is needed to create NLU section of report. (default: {constants.RASA_API_URL})" ) @click.option( "--rasa-version", type=str, required=False, - default=None, + default=constants.RASA_VERSION, help="Rasa version. It's only displayed in the report for project documentation." ) @click.version_option( @@ -95,6 +104,7 @@ def main( model_link, no_images, output_path, path, + precision, project_name, project_version, rasa_api, @@ -113,6 +123,7 @@ def main( rasa_api_url=rasa_api, model_link=model_link, actions_path=actions_path, - no_images=no_images + no_images=no_images, + precision=precision ) return report diff --git a/tests/test_json_controller.py b/tests/test_json_controller.py index 0fec4cd..ab23765 100644 --- a/tests/test_json_controller.py +++ b/tests/test_json_controller.py @@ -84,12 +84,12 @@ def test_load_overview(): assert isinstance(overview.get("project"), str) assert isinstance(overview.get("version"), str) assert isinstance(overview.get("updated_at"), str) - assert isinstance(overview.get("intent"), float) - assert isinstance(overview.get("entity"), float) - assert isinstance(overview.get("core"), float) - assert isinstance(overview.get("nlu"), float) or overview.get("nlu") is None - assert isinstance(overview.get("e2e_coverage"), float) or overview.get("e2e_coverage") is None - assert isinstance(overview.get("overall"), float) + assert isinstance(overview.get("intent"), (float, int)) + assert isinstance(overview.get("entity"), (float, int)) + assert isinstance(overview.get("core"), (float, int)) + assert isinstance(overview.get("nlu"), (float, int)) or overview.get("nlu") is None + assert isinstance(overview.get("e2e_coverage"), (float, int)) or overview.get("e2e_coverage") is None + assert isinstance(overview.get("overall"), (float, int)) assert isinstance(overview.get("created_at"), str) @@ -110,7 +110,7 @@ def test_save_overview(): def test_calculate_overall(): json_controller = pytest.json_controller json_controller._calculate_overall() - assert isinstance(json_controller.overview.get("overall"), float) + assert isinstance(json_controller.overview.get("overall"), (float, int)) def test_update_overview(): diff --git a/tests/test_markdown_controller.py b/tests/test_markdown_controller.py index 6910dd1..1eff9df 100644 --- a/tests/test_markdown_controller.py +++ b/tests/test_markdown_controller.py @@ -86,6 +86,7 @@ def test_build_summary(): def test_build_summary_without_nlu_section(): markdown_controller = pytest.markdown_controller + markdown_controller.nlu._connected = False text = markdown_controller.build_summary() assert isinstance(text, str) assert markdown_controller.nlu.is_connected() is False @@ -257,8 +258,7 @@ def test_build_nlu_table(): def test_build_nlu_table_if_len_less_than_2(): markdown_controller = pytest.markdown_controller - json_controller = JsonController("invelid/path", "./", "test-project", "0.0.0") - markdown_controller.json = json_controller + markdown_controller.nlu._data = {} text = markdown_controller.build_nlu_table() assert isinstance(text, str) assert "No example sentences were found in this template" in text @@ -280,8 +280,7 @@ def test_build_nlu_errors_table(): def test_build_nlu_errors_table_if_len_less_than_2(): markdown_controller = pytest.markdown_controller - json_controller = JsonController("invalid/path", "./", "test-project", "0.0.0") - markdown_controller.json = json_controller + markdown_controller.nlu._problem_sentences = {} text = markdown_controller.build_nlu_errors_table() assert isinstance(text, str) assert "There are no sentences that were not understood in this model" in text diff --git a/tests/test_utils.py b/tests/test_utils.py index 37efa16..76775d1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -37,17 +37,49 @@ def test_get_project_name(): def test_change_scale(): assert isinstance(change_scale(0.123312, 10), str) - assert change_scale(0.123312, 10) == "1" - assert change_scale(98.6, 1) == "98" - assert change_scale(0.4629, 100) == "46" - assert change_scale(90.5, 0.1) == "9" + assert change_scale(0.123312, 10) == "1.2" + assert change_scale(98.6, 1) == "98.6" + assert change_scale(0.4629, 100) == "46.3" assert change_scale(0.001) == "0" - assert change_scale(None) is None + assert change_scale(0.3) == "0.3" + assert change_scale(0.09) == "0.1" + assert change_scale(90.5, 0.1) == "9.1" + assert change_scale(0.001, 0.1) == "0" + + # Other precisions + assert change_scale(10, 1, 2) == "10" + assert change_scale(39.591231, 1, 2) == "39.59" + assert change_scale(0.05281239, 1, 1) == "0.1" + assert change_scale(0.5219483, 10, 2) == "5.22" + assert change_scale(0.4, 100, 3) == "40" + assert change_scale(0.511232, 100, 3) == "51.123" + assert change_scale(0.85, 100, 2) == "85" + assert change_scale(0.12239432, 100, 5) == "12.23943" + assert change_scale(19.51, 1, 4) == "19.5100" + assert change_scale(19.51, 1, 0) == "20" + assert change_scale(0.56831, 100, 0) == "57" + assert change_scale(0.731, 10, 0) == "7" + + # Invalid scales + assert change_scale(10, 0) == 10 + assert change_scale(0.001, "test") == 0.001 + + # Invalid values assert change_scale("-") == "-" assert change_scale("test", 100) == "test" + assert change_scale(None) is None + assert change_scale("100", 1) == "100" + + # Invalid precisions + assert change_scale(0.85, 100, None) == "85" + assert change_scale(123, 1, "2") == "123" + assert change_scale(59, 1, 0.5) == "59" + assert change_scale(1.8473, 1, 10) == "1.8" + assert change_scale(9.123123, 1, 6) == "9.1" def test_get_color(): + assert get_color(10, 10) == "🟢" assert get_color(0.98) == "🟢" assert get_color(0.9012) == "🟢" assert get_color(0.899) == "🟡" @@ -66,8 +98,12 @@ def test_get_color(): assert get_color(0.0009) == "❌" assert get_color(0) == "❌" assert get_color(0, 100) == "❌" + + # Invalid values assert get_color(-1) == "❌" assert get_color(None) == "❌" + assert get_color("10", 10) == "❌" + assert get_color("1") == "❌" assert get_color("-") == "❌" assert get_color("test", 100) == "❌"