#63 - New parameter --precision

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.3.3] - In progress...
+### Added
+- [#63](https://github.com/brunohjs/rasa-model-report/issues/63) Created `--precision` CLI command parameter. This command is used to change precision of the model report overview grades.
+
 ## [1.3.2] - 2023-02-26
 ### Added
 - [#26](https://github.com/brunohjs/rasa-model-report/issues/26) Updated release script.
@@ -70,6 +74,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - [#16](https://github.com/brunohjs/rasa-model-report/issues/16) Created a handler for retrieval intents in the report.
 
+[1.3.3]: https://github.com/brunohjs/rasa-model-report/compare/1.3.2...HEAD
 [1.3.2]: https://github.com/brunohjs/rasa-model-report/compare/1.3.0...1.3.2
 [1.3.1]: https://github.com/brunohjs/rasa-model-report/compare/1.3.0...1.3.1
 [1.3.0]: https://github.com/brunohjs/rasa-model-report/compare/1.2.0...1.3.0

README.md

@@ -93,14 +93,15 @@ There are parameters that can be used. Available options are below:
                         path)
 --disable-nlu           Disable processing NLU sentences. NLU section will
                         not be generated in the report. Required Rasa API.
-                        (default: false)
 -h, --help              Show this help message.
 --model-link TEXT       Model download link. It's only displayed in the
                         report to model download.
---no-images             Generate model report without images. (default:
-                        false)
+--no-images             Generate model report without images.
 --output-path TEXT      Report output path. (default: ./)
 -p, --path TEXT         Rasa project path. (default: ./)
+--precision INTEGER     Grade precision. Used to change precision of the
+                        model report overview grades. Can vary between 0 and
+                        5 (default: 1)
 --project-name TEXT     Rasa project name. It's only displayed in the
                         report. (default: My project)
 --project-version TEXT  Project version. It's only displayed in the report

rasa_model_report/controllers/json_controller.py

@@ -216,6 +216,10 @@ def save_overview(self) -> None:
         file.write("\n")
         file.close()
 
+    @staticmethod
+    def weight_function(x: float) -> float:
+        return 1 - x**2
+
     def _calculate_overall(self) -> None:
         """
         Calculate report overall.
@@ -224,15 +228,15 @@ def _calculate_overall(self) -> None:
             "intent": 2,
             "entity": 1,
             "core": 1,
-            "nlu": 3,
-            "e2e_coverage": 3
+            "nlu": 8,
+            "e2e_coverage": 8
         }
         overview_sum = sum(
-            self._overview[item] * w for item, w in weights.items()
+            self._overview[item] * self.weight_function(self._overview[item]) for item in weights
             if isinstance(self._overview[item], (int, float)) and self._overview[item] >= 0
         )
         weight_sum = sum(
-            w for item, w in weights.items()
+            self.weight_function(self._overview[item]) for item in weights
             if isinstance(self._overview[item], (int, float)) and self._overview[item] >= 0
         )
         overview_rate = overview_sum / weight_sum if weight_sum else 0

rasa_model_report/controllers/markdown_controller.py

@@ -10,6 +10,7 @@
 from rasa_model_report.controllers.e2e_coverage_controller import E2ECoverageController
 from rasa_model_report.controllers.json_controller import JsonController
 from rasa_model_report.controllers.nlu_controller import NluController
+from rasa_model_report.helpers import constants
 from rasa_model_report.helpers import type_aliases
 from rasa_model_report.helpers import utils
 
@@ -44,16 +45,17 @@ def __init__(
         self.output_report_path: str = utils.remove_duplicate_slashs(f"{self.output_path}/model_report.md")
         self.readme_path: str = "README.md"
         self.model_link: str = kwargs.get("model_link")
-        self.no_images: bool = kwargs.get("no_images", False)
+        self.no_images: bool = kwargs.get("no_images", constants.NO_IMAGES)
+        self.precision: int = kwargs.get("precision", constants.GRADE_PRECISION)
         self.json: JsonController = JsonController(rasa_path, output_path, project_name, project_version)
         self.csv: CsvController = CsvController(rasa_path, output_path, project_name, project_version)
         self.nlu: NluController = NluController(
             rasa_path,
             output_path,
             project_name,
             project_version,
-            url=kwargs.get("rasa_api_url"),
-            disable_nlu=kwargs.get("disable_nlu")
+            url=kwargs.get("rasa_api_url", constants.RASA_API_URL),
+            disable_nlu=kwargs.get("disable_nlu", constants.DISABLE_NLU)
         )
         self.e2e_coverage: E2ECoverageController = E2ECoverageController(
             rasa_path,
@@ -182,12 +184,12 @@ def build_overview(self) -> str:
         style = "style='font-size:20px'"
         text += f"|Intent|Entity|NLU|Core|E2E Coverage|<span {style}>General</span>|\n"
         text += "|:-:|:-:|:-:|:-:|:-:|:-:|\n"
-        text += f"|{utils.change_scale(overview['intent'], 10)}\
-            |{utils.change_scale(overview['entity'], 10)}\
-            |{utils.change_scale(overview['nlu'], 10)}\
-            |{utils.change_scale(overview['core'], 10)}\
-            |{utils.change_scale(overview['e2e_coverage'], 10)}\
-            |<span {style}>**{utils.change_scale(overview['overall'], 10)}**</span>|\n"
+        text += f"|{utils.change_scale(overview['intent'], 10, self.precision)}\
+            |{utils.change_scale(overview['entity'], 10, self.precision)}\
+            |{utils.change_scale(overview['nlu'], 10, self.precision)}\
+            |{utils.change_scale(overview['core'], 10, self.precision)}\
+            |{utils.change_scale(overview['e2e_coverage'], 10, self.precision)}\
+            |<span {style}>**{utils.change_scale(overview['overall'], 10, self.precision)}**</span>|\n"
         text += f"{utils.get_color(overview['intent'])}\
             |{utils.get_color(overview['entity'])}\
             |{utils.get_color(overview['nlu'])}\

rasa_model_report/controllers/nlu_controller.py

@@ -10,6 +10,7 @@
 import requests.exceptions
 
 from rasa_model_report.controllers.controller import Controller
+from rasa_model_report.helpers import constants
 from rasa_model_report.helpers import type_aliases
 from rasa_model_report.helpers import utils
 
@@ -41,7 +42,7 @@ def __init__(
         self._problem_sentences: List[type_aliases.nlu_payload] = []
         self._general_grade: Optional[float] = None
         self._connected: bool = False
-        self._disable_nlu: bool = kwargs.get("disable_nlu")
+        self._disable_nlu: bool = kwargs.get("disable_nlu", constants.DISABLE_NLU)
         self.url: str = url
 
         if not self._disable_nlu and self.health_check_rasa_api():
@@ -108,7 +109,7 @@ def _generate_data(self) -> List[type_aliases.nlu_payload]:
         for intent, examples in self._data.items():
             progress = index / len(self._data) * 100
             index += 1
-            logging.info(f" - Analyzing NLU of the {intent} intent ({progress:<5.1f}%).")
+            logging.info(f" - ({progress:<5.1f}%) analyzing NLU intent: {intent}")
             for text in examples:
                 text = self.remove_entities_from_text(text)
                 nlu_requested = self.request_nlu(text)

rasa_model_report/helpers/constants.py

@@ -0,0 +1,9 @@
+DISABLE_NLU = False
+GRADE_PRECISION = 1
+NO_IMAGES = False
+OUTPUT_PATH = "./"
+PROJECT_NAME = "My project"
+PROJECT_VERSION = None
+RASA_API_URL = "http://localhost:5005"
+RASA_PATH = "./"
+RASA_VERSION = None

rasa_model_report/helpers/type_aliases.py

@@ -2,6 +2,7 @@
 from typing import List
 from typing import Union
 
+
 intent = Dict[str, Union[float, str, None]]
 entity = Dict[str, Union[float, Dict[str, float], None]]
 nlu_payload = Dict[str, Union[intent, List[intent], str, float, None]]

rasa_model_report/helpers/utils.py

@@ -11,6 +11,8 @@
 from requests.adapters import Retry
 from yaml import safe_load
 
+from rasa_model_report.helpers import constants
+
 
 def format_date() -> str:
     """
@@ -64,16 +66,28 @@ def get_color(value: float, scale: int = 1) -> str:
     return "❌"
 
 
-def change_scale(value: float, scale: int = 1) -> str:
+def change_scale(value: float, scale: int = 1, precision: int = 1) -> str:
     """
     Change the value scale and rounds it to display in string format.
 
     :param value: Value that will be changed to scale and rounds it.
     :param scale: Scale that will be applied.
-    :return: Value on the new scale.
-    """
-    if isinstance(value, (float, int)):
-        return f"{int(value * scale)}"
+    :param precision: Value precision.
+    :return: Value on the new scale and precision.
+    """
+    precision = precision if isinstance(precision, int) and 5 >= precision >= 0 else constants.GRADE_PRECISION
+    if (
+        isinstance(value, (float, int)) and
+        isinstance(scale, (float, int)) and
+        scale != 0
+    ):
+        new_value = value * scale
+        if new_value >= 1 and new_value % int(new_value) == 0:
+            return str(int(new_value))
+        elif re.search(r"\.0$", f"{new_value:.{precision}f}"):
+            return "0"
+        else:
+            return f"{new_value:.{precision}f}"
     return value
 
 

rasa_model_report/main.py

@@ -3,6 +3,7 @@
 import click
 
 from rasa_model_report.controllers.model_report import ModelReport
+from rasa_model_report.helpers import constants
 
 logging.basicConfig(format="%(asctime)s [%(levelname)s] %(message)s", level=logging.INFO)
 
@@ -17,9 +18,9 @@
     "--disable-nlu",
     is_flag=True,
     required=False,
-    default=False,
+    default=constants.DISABLE_NLU,
     help="Disable processing NLU sentences. NLU section will not be generated "
-    "in the report. Required Rasa API. (default: false)"
+    "in the report. Required Rasa API."
 )
 @click.help_option(
     "--help",
@@ -36,50 +37,58 @@
     "--no-images",
     is_flag=True,
     required=False,
-    default=False,
-    help="Generate model report without images. (default: false)"
+    default=constants.NO_IMAGES,
+    help="Generate model report without images."
 )
 @click.option(
     "--output-path",
     type=str,
     required=False,
-    default="./",
-    help="Report output path. (default: ./)"
+    default=constants.OUTPUT_PATH,
+    help=f"Report output path. (default: {constants.OUTPUT_PATH})"
 )
 @click.option(
     "--path",
     "-p",
     type=str,
     required=False,
-    default="./",
-    help="Rasa project path. (default: ./)"
+    default=constants.RASA_PATH,
+    help=f"Rasa project path. (default: {constants.RASA_PATH})"
+)
+@click.option(
+    "--precision",
+    type=int,
+    required=False,
+    default=constants.GRADE_PRECISION,
+    help="Grade precision. Used to change precision of the model report overview grades. "
+    f"Can vary between 0 and 5 (default: {constants.GRADE_PRECISION})"
 )
 @click.option(
     "--project-name",
     type=str,
     required=False,
-    default="My Project",
-    help="Rasa project name. It's only displayed in the report. (default: My project)"
+    default=constants.PROJECT_NAME,
+    help=f"Rasa project name. It's only displayed in the report. (default: {constants.PROJECT_NAME})"
 )
 @click.option(
     "--project-version",
     type=str,
     required=False,
-    default=None,
+    default=constants.PROJECT_VERSION,
     help="Project version. It's only displayed in the report for project documentation."
 )
 @click.option(
     "--rasa-api",
     type=str,
     required=False,
-    default="http://localhost:5005",
-    help="Rasa API URL. Is needed to create NLU section of report. (default: http://localhost:5005)"
+    default=constants.RASA_API_URL,
+    help=f"Rasa API URL. Is needed to create NLU section of report. (default: {constants.RASA_API_URL})"
 )
 @click.option(
     "--rasa-version",
     type=str,
     required=False,
-    default=None,
+    default=constants.RASA_VERSION,
     help="Rasa version. It's only displayed in the report for project documentation."
 )
 @click.version_option(
@@ -95,6 +104,7 @@ def main(
     model_link,
     no_images,
     output_path, path,
+    precision,
     project_name,
     project_version,
     rasa_api,
@@ -113,6 +123,7 @@ def main(
         rasa_api_url=rasa_api,
         model_link=model_link,
         actions_path=actions_path,
-        no_images=no_images
+        no_images=no_images,
+        precision=precision
     )
     return report

tests/test_json_controller.py

@@ -84,12 +84,12 @@ def test_load_overview():
     assert isinstance(overview.get("project"), str)
     assert isinstance(overview.get("version"), str)
     assert isinstance(overview.get("updated_at"), str)
-    assert isinstance(overview.get("intent"), float)
-    assert isinstance(overview.get("entity"), float)
-    assert isinstance(overview.get("core"), float)
-    assert isinstance(overview.get("nlu"), float) or overview.get("nlu") is None
-    assert isinstance(overview.get("e2e_coverage"), float) or overview.get("e2e_coverage") is None
-    assert isinstance(overview.get("overall"), float)
+    assert isinstance(overview.get("intent"), (float, int))
+    assert isinstance(overview.get("entity"), (float, int))
+    assert isinstance(overview.get("core"), (float, int))
+    assert isinstance(overview.get("nlu"), (float, int)) or overview.get("nlu") is None
+    assert isinstance(overview.get("e2e_coverage"), (float, int)) or overview.get("e2e_coverage") is None
+    assert isinstance(overview.get("overall"), (float, int))
     assert isinstance(overview.get("created_at"), str)
 
 
@@ -110,7 +110,7 @@ def test_save_overview():
 def test_calculate_overall():
     json_controller = pytest.json_controller
     json_controller._calculate_overall()
-    assert isinstance(json_controller.overview.get("overall"), float)
+    assert isinstance(json_controller.overview.get("overall"), (float, int))
 
 
 def test_update_overview():

tests/test_markdown_controller.py

@@ -86,6 +86,7 @@ def test_build_summary():
 
 def test_build_summary_without_nlu_section():
     markdown_controller = pytest.markdown_controller
+    markdown_controller.nlu._connected = False
     text = markdown_controller.build_summary()
     assert isinstance(text, str)
     assert markdown_controller.nlu.is_connected() is False
@@ -257,8 +258,7 @@ def test_build_nlu_table():
 
 def test_build_nlu_table_if_len_less_than_2():
     markdown_controller = pytest.markdown_controller
-    json_controller = JsonController("invelid/path", "./", "test-project", "0.0.0")
-    markdown_controller.json = json_controller
+    markdown_controller.nlu._data = {}
     text = markdown_controller.build_nlu_table()
     assert isinstance(text, str)
     assert "No example sentences were found in this template" in text
@@ -280,8 +280,7 @@ def test_build_nlu_errors_table():
 
 def test_build_nlu_errors_table_if_len_less_than_2():
     markdown_controller = pytest.markdown_controller
-    json_controller = JsonController("invalid/path", "./", "test-project", "0.0.0")
-    markdown_controller.json = json_controller
+    markdown_controller.nlu._problem_sentences = {}
     text = markdown_controller.build_nlu_errors_table()
     assert isinstance(text, str)
     assert "There are no sentences that were not understood in this model" in text