diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01c0b0c..0742908 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [1.3.3] - In progress...
+### Added
+- [#63](https://github.com/brunohjs/rasa-model-report/issues/63) Created `--precision` CLI command parameter. This command is used to change precision of the model report overview grades.
+
## [1.3.2] - 2023-02-26
### Added
- [#26](https://github.com/brunohjs/rasa-model-report/issues/26) Updated release script.
@@ -70,6 +74,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- [#16](https://github.com/brunohjs/rasa-model-report/issues/16) Created a handler for retrieval intents in the report.
+[1.3.3]: https://github.com/brunohjs/rasa-model-report/compare/1.3.2...HEAD
[1.3.2]: https://github.com/brunohjs/rasa-model-report/compare/1.3.0...1.3.2
[1.3.1]: https://github.com/brunohjs/rasa-model-report/compare/1.3.0...1.3.1
[1.3.0]: https://github.com/brunohjs/rasa-model-report/compare/1.2.0...1.3.0
diff --git a/README.md b/README.md
index adc333d..6f68860 100644
--- a/README.md
+++ b/README.md
@@ -93,14 +93,15 @@ There are parameters that can be used. Available options are below:
path)
--disable-nlu Disable processing NLU sentences. NLU section will
not be generated in the report. Required Rasa API.
- (default: false)
-h, --help Show this help message.
--model-link TEXT Model download link. It's only displayed in the
report to model download.
---no-images Generate model report without images. (default:
- false)
+--no-images Generate model report without images.
--output-path TEXT Report output path. (default: ./)
-p, --path TEXT Rasa project path. (default: ./)
+--precision INTEGER Grade precision. Used to change precision of the
+ model report overview grades. Can vary between 0 and
+ 5 (default: 1)
--project-name TEXT Rasa project name. It's only displayed in the
report. (default: My project)
--project-version TEXT Project version. It's only displayed in the report
diff --git a/rasa_model_report/controllers/json_controller.py b/rasa_model_report/controllers/json_controller.py
index 9b05208..881f818 100644
--- a/rasa_model_report/controllers/json_controller.py
+++ b/rasa_model_report/controllers/json_controller.py
@@ -216,6 +216,10 @@ def save_overview(self) -> None:
file.write("\n")
file.close()
+ @staticmethod
+ def weight_function(x: float) -> float:
+ return 1 - x**2
+
def _calculate_overall(self) -> None:
"""
Calculate report overall.
@@ -224,15 +228,15 @@ def _calculate_overall(self) -> None:
"intent": 2,
"entity": 1,
"core": 1,
- "nlu": 3,
- "e2e_coverage": 3
+ "nlu": 8,
+ "e2e_coverage": 8
}
overview_sum = sum(
- self._overview[item] * w for item, w in weights.items()
+ self._overview[item] * self.weight_function(self._overview[item]) for item in weights
if isinstance(self._overview[item], (int, float)) and self._overview[item] >= 0
)
weight_sum = sum(
- w for item, w in weights.items()
+ self.weight_function(self._overview[item]) for item in weights
if isinstance(self._overview[item], (int, float)) and self._overview[item] >= 0
)
overview_rate = overview_sum / weight_sum if weight_sum else 0
diff --git a/rasa_model_report/controllers/markdown_controller.py b/rasa_model_report/controllers/markdown_controller.py
index fcb5d3b..b3f2cd3 100644
--- a/rasa_model_report/controllers/markdown_controller.py
+++ b/rasa_model_report/controllers/markdown_controller.py
@@ -10,6 +10,7 @@
from rasa_model_report.controllers.e2e_coverage_controller import E2ECoverageController
from rasa_model_report.controllers.json_controller import JsonController
from rasa_model_report.controllers.nlu_controller import NluController
+from rasa_model_report.helpers import constants
from rasa_model_report.helpers import type_aliases
from rasa_model_report.helpers import utils
@@ -44,7 +45,8 @@ def __init__(
self.output_report_path: str = utils.remove_duplicate_slashs(f"{self.output_path}/model_report.md")
self.readme_path: str = "README.md"
self.model_link: str = kwargs.get("model_link")
- self.no_images: bool = kwargs.get("no_images", False)
+ self.no_images: bool = kwargs.get("no_images", constants.NO_IMAGES)
+ self.precision: int = kwargs.get("precision", constants.GRADE_PRECISION)
self.json: JsonController = JsonController(rasa_path, output_path, project_name, project_version)
self.csv: CsvController = CsvController(rasa_path, output_path, project_name, project_version)
self.nlu: NluController = NluController(
@@ -52,8 +54,8 @@ def __init__(
output_path,
project_name,
project_version,
- url=kwargs.get("rasa_api_url"),
- disable_nlu=kwargs.get("disable_nlu")
+ url=kwargs.get("rasa_api_url", constants.RASA_API_URL),
+ disable_nlu=kwargs.get("disable_nlu", constants.DISABLE_NLU)
)
self.e2e_coverage: E2ECoverageController = E2ECoverageController(
rasa_path,
@@ -182,12 +184,12 @@ def build_overview(self) -> str:
style = "style='font-size:20px'"
text += f"|Intent|Entity|NLU|Core|E2E Coverage|General|\n"
text += "|:-:|:-:|:-:|:-:|:-:|:-:|\n"
- text += f"|{utils.change_scale(overview['intent'], 10)}\
- |{utils.change_scale(overview['entity'], 10)}\
- |{utils.change_scale(overview['nlu'], 10)}\
- |{utils.change_scale(overview['core'], 10)}\
- |{utils.change_scale(overview['e2e_coverage'], 10)}\
- |**{utils.change_scale(overview['overall'], 10)}**|\n"
+ text += f"|{utils.change_scale(overview['intent'], 10, self.precision)}\
+ |{utils.change_scale(overview['entity'], 10, self.precision)}\
+ |{utils.change_scale(overview['nlu'], 10, self.precision)}\
+ |{utils.change_scale(overview['core'], 10, self.precision)}\
+ |{utils.change_scale(overview['e2e_coverage'], 10, self.precision)}\
+ |**{utils.change_scale(overview['overall'], 10, self.precision)}**|\n"
text += f"{utils.get_color(overview['intent'])}\
|{utils.get_color(overview['entity'])}\
|{utils.get_color(overview['nlu'])}\
diff --git a/rasa_model_report/controllers/nlu_controller.py b/rasa_model_report/controllers/nlu_controller.py
index 95b8b34..8cb3435 100644
--- a/rasa_model_report/controllers/nlu_controller.py
+++ b/rasa_model_report/controllers/nlu_controller.py
@@ -10,6 +10,7 @@
import requests.exceptions
from rasa_model_report.controllers.controller import Controller
+from rasa_model_report.helpers import constants
from rasa_model_report.helpers import type_aliases
from rasa_model_report.helpers import utils
@@ -41,7 +42,7 @@ def __init__(
self._problem_sentences: List[type_aliases.nlu_payload] = []
self._general_grade: Optional[float] = None
self._connected: bool = False
- self._disable_nlu: bool = kwargs.get("disable_nlu")
+ self._disable_nlu: bool = kwargs.get("disable_nlu", constants.DISABLE_NLU)
self.url: str = url
if not self._disable_nlu and self.health_check_rasa_api():
@@ -108,7 +109,7 @@ def _generate_data(self) -> List[type_aliases.nlu_payload]:
for intent, examples in self._data.items():
progress = index / len(self._data) * 100
index += 1
- logging.info(f" - Analyzing NLU of the {intent} intent ({progress:<5.1f}%).")
+ logging.info(f" - ({progress:<5.1f}%) analyzing NLU intent: {intent}")
for text in examples:
text = self.remove_entities_from_text(text)
nlu_requested = self.request_nlu(text)
diff --git a/rasa_model_report/helpers/constants.py b/rasa_model_report/helpers/constants.py
new file mode 100644
index 0000000..7cccd7a
--- /dev/null
+++ b/rasa_model_report/helpers/constants.py
@@ -0,0 +1,9 @@
+DISABLE_NLU = False
+GRADE_PRECISION = 1
+NO_IMAGES = False
+OUTPUT_PATH = "./"
+PROJECT_NAME = "My project"
+PROJECT_VERSION = None
+RASA_API_URL = "http://localhost:5005"
+RASA_PATH = "./"
+RASA_VERSION = None
diff --git a/rasa_model_report/helpers/type_aliases.py b/rasa_model_report/helpers/type_aliases.py
index a61db1f..b52b339 100644
--- a/rasa_model_report/helpers/type_aliases.py
+++ b/rasa_model_report/helpers/type_aliases.py
@@ -2,6 +2,7 @@
from typing import List
from typing import Union
+
intent = Dict[str, Union[float, str, None]]
entity = Dict[str, Union[float, Dict[str, float], None]]
nlu_payload = Dict[str, Union[intent, List[intent], str, float, None]]
diff --git a/rasa_model_report/helpers/utils.py b/rasa_model_report/helpers/utils.py
index 4aca15d..82ab37e 100644
--- a/rasa_model_report/helpers/utils.py
+++ b/rasa_model_report/helpers/utils.py
@@ -11,6 +11,8 @@
from requests.adapters import Retry
from yaml import safe_load
+from rasa_model_report.helpers import constants
+
def format_date() -> str:
"""
@@ -64,16 +66,28 @@ def get_color(value: float, scale: int = 1) -> str:
return "❌"
-def change_scale(value: float, scale: int = 1) -> str:
+def change_scale(value: float, scale: int = 1, precision: int = 1) -> str:
"""
Change the value scale and rounds it to display in string format.
:param value: Value that will be changed to scale and rounds it.
:param scale: Scale that will be applied.
- :return: Value on the new scale.
- """
- if isinstance(value, (float, int)):
- return f"{int(value * scale)}"
+ :param precision: Value precision.
+ :return: Value on the new scale and precision.
+ """
+ precision = precision if isinstance(precision, int) and 5 >= precision >= 0 else constants.GRADE_PRECISION
+ if (
+ isinstance(value, (float, int)) and
+ isinstance(scale, (float, int)) and
+ scale != 0
+ ):
+ new_value = value * scale
+ if new_value >= 1 and new_value % int(new_value) == 0:
+ return str(int(new_value))
+ elif re.search(r"\.0$", f"{new_value:.{precision}f}"):
+ return "0"
+ else:
+ return f"{new_value:.{precision}f}"
return value
diff --git a/rasa_model_report/main.py b/rasa_model_report/main.py
index f67a2fe..45312bb 100644
--- a/rasa_model_report/main.py
+++ b/rasa_model_report/main.py
@@ -3,6 +3,7 @@
import click
from rasa_model_report.controllers.model_report import ModelReport
+from rasa_model_report.helpers import constants
logging.basicConfig(format="%(asctime)s [%(levelname)s] %(message)s", level=logging.INFO)
@@ -17,9 +18,9 @@
"--disable-nlu",
is_flag=True,
required=False,
- default=False,
+ default=constants.DISABLE_NLU,
help="Disable processing NLU sentences. NLU section will not be generated "
- "in the report. Required Rasa API. (default: false)"
+ "in the report. Required Rasa API."
)
@click.help_option(
"--help",
@@ -36,50 +37,58 @@
"--no-images",
is_flag=True,
required=False,
- default=False,
- help="Generate model report without images. (default: false)"
+ default=constants.NO_IMAGES,
+ help="Generate model report without images."
)
@click.option(
"--output-path",
type=str,
required=False,
- default="./",
- help="Report output path. (default: ./)"
+ default=constants.OUTPUT_PATH,
+ help=f"Report output path. (default: {constants.OUTPUT_PATH})"
)
@click.option(
"--path",
"-p",
type=str,
required=False,
- default="./",
- help="Rasa project path. (default: ./)"
+ default=constants.RASA_PATH,
+ help=f"Rasa project path. (default: {constants.RASA_PATH})"
+)
+@click.option(
+ "--precision",
+ type=int,
+ required=False,
+ default=constants.GRADE_PRECISION,
+ help="Grade precision. Used to change precision of the model report overview grades. "
+ f"Can vary between 0 and 5 (default: {constants.GRADE_PRECISION})"
)
@click.option(
"--project-name",
type=str,
required=False,
- default="My Project",
- help="Rasa project name. It's only displayed in the report. (default: My project)"
+ default=constants.PROJECT_NAME,
+ help=f"Rasa project name. It's only displayed in the report. (default: {constants.PROJECT_NAME})"
)
@click.option(
"--project-version",
type=str,
required=False,
- default=None,
+ default=constants.PROJECT_VERSION,
help="Project version. It's only displayed in the report for project documentation."
)
@click.option(
"--rasa-api",
type=str,
required=False,
- default="http://localhost:5005",
- help="Rasa API URL. Is needed to create NLU section of report. (default: http://localhost:5005)"
+ default=constants.RASA_API_URL,
+ help=f"Rasa API URL. Is needed to create NLU section of report. (default: {constants.RASA_API_URL})"
)
@click.option(
"--rasa-version",
type=str,
required=False,
- default=None,
+ default=constants.RASA_VERSION,
help="Rasa version. It's only displayed in the report for project documentation."
)
@click.version_option(
@@ -95,6 +104,7 @@ def main(
model_link,
no_images,
output_path, path,
+ precision,
project_name,
project_version,
rasa_api,
@@ -113,6 +123,7 @@ def main(
rasa_api_url=rasa_api,
model_link=model_link,
actions_path=actions_path,
- no_images=no_images
+ no_images=no_images,
+ precision=precision
)
return report
diff --git a/tests/test_json_controller.py b/tests/test_json_controller.py
index 0fec4cd..ab23765 100644
--- a/tests/test_json_controller.py
+++ b/tests/test_json_controller.py
@@ -84,12 +84,12 @@ def test_load_overview():
assert isinstance(overview.get("project"), str)
assert isinstance(overview.get("version"), str)
assert isinstance(overview.get("updated_at"), str)
- assert isinstance(overview.get("intent"), float)
- assert isinstance(overview.get("entity"), float)
- assert isinstance(overview.get("core"), float)
- assert isinstance(overview.get("nlu"), float) or overview.get("nlu") is None
- assert isinstance(overview.get("e2e_coverage"), float) or overview.get("e2e_coverage") is None
- assert isinstance(overview.get("overall"), float)
+ assert isinstance(overview.get("intent"), (float, int))
+ assert isinstance(overview.get("entity"), (float, int))
+ assert isinstance(overview.get("core"), (float, int))
+ assert isinstance(overview.get("nlu"), (float, int)) or overview.get("nlu") is None
+ assert isinstance(overview.get("e2e_coverage"), (float, int)) or overview.get("e2e_coverage") is None
+ assert isinstance(overview.get("overall"), (float, int))
assert isinstance(overview.get("created_at"), str)
@@ -110,7 +110,7 @@ def test_save_overview():
def test_calculate_overall():
json_controller = pytest.json_controller
json_controller._calculate_overall()
- assert isinstance(json_controller.overview.get("overall"), float)
+ assert isinstance(json_controller.overview.get("overall"), (float, int))
def test_update_overview():
diff --git a/tests/test_markdown_controller.py b/tests/test_markdown_controller.py
index 6910dd1..1eff9df 100644
--- a/tests/test_markdown_controller.py
+++ b/tests/test_markdown_controller.py
@@ -86,6 +86,7 @@ def test_build_summary():
def test_build_summary_without_nlu_section():
markdown_controller = pytest.markdown_controller
+ markdown_controller.nlu._connected = False
text = markdown_controller.build_summary()
assert isinstance(text, str)
assert markdown_controller.nlu.is_connected() is False
@@ -257,8 +258,7 @@ def test_build_nlu_table():
def test_build_nlu_table_if_len_less_than_2():
markdown_controller = pytest.markdown_controller
- json_controller = JsonController("invelid/path", "./", "test-project", "0.0.0")
- markdown_controller.json = json_controller
+ markdown_controller.nlu._data = {}
text = markdown_controller.build_nlu_table()
assert isinstance(text, str)
assert "No example sentences were found in this template" in text
@@ -280,8 +280,7 @@ def test_build_nlu_errors_table():
def test_build_nlu_errors_table_if_len_less_than_2():
markdown_controller = pytest.markdown_controller
- json_controller = JsonController("invalid/path", "./", "test-project", "0.0.0")
- markdown_controller.json = json_controller
+ markdown_controller.nlu._problem_sentences = {}
text = markdown_controller.build_nlu_errors_table()
assert isinstance(text, str)
assert "There are no sentences that were not understood in this model" in text
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 37efa16..76775d1 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -37,17 +37,49 @@ def test_get_project_name():
def test_change_scale():
assert isinstance(change_scale(0.123312, 10), str)
- assert change_scale(0.123312, 10) == "1"
- assert change_scale(98.6, 1) == "98"
- assert change_scale(0.4629, 100) == "46"
- assert change_scale(90.5, 0.1) == "9"
+ assert change_scale(0.123312, 10) == "1.2"
+ assert change_scale(98.6, 1) == "98.6"
+ assert change_scale(0.4629, 100) == "46.3"
assert change_scale(0.001) == "0"
- assert change_scale(None) is None
+ assert change_scale(0.3) == "0.3"
+ assert change_scale(0.09) == "0.1"
+ assert change_scale(90.5, 0.1) == "9.1"
+ assert change_scale(0.001, 0.1) == "0"
+
+ # Other precisions
+ assert change_scale(10, 1, 2) == "10"
+ assert change_scale(39.591231, 1, 2) == "39.59"
+ assert change_scale(0.05281239, 1, 1) == "0.1"
+ assert change_scale(0.5219483, 10, 2) == "5.22"
+ assert change_scale(0.4, 100, 3) == "40"
+ assert change_scale(0.511232, 100, 3) == "51.123"
+ assert change_scale(0.85, 100, 2) == "85"
+ assert change_scale(0.12239432, 100, 5) == "12.23943"
+ assert change_scale(19.51, 1, 4) == "19.5100"
+ assert change_scale(19.51, 1, 0) == "20"
+ assert change_scale(0.56831, 100, 0) == "57"
+ assert change_scale(0.731, 10, 0) == "7"
+
+ # Invalid scales
+ assert change_scale(10, 0) == 10
+ assert change_scale(0.001, "test") == 0.001
+
+ # Invalid values
assert change_scale("-") == "-"
assert change_scale("test", 100) == "test"
+ assert change_scale(None) is None
+ assert change_scale("100", 1) == "100"
+
+ # Invalid precisions
+ assert change_scale(0.85, 100, None) == "85"
+ assert change_scale(123, 1, "2") == "123"
+ assert change_scale(59, 1, 0.5) == "59"
+ assert change_scale(1.8473, 1, 10) == "1.8"
+ assert change_scale(9.123123, 1, 6) == "9.1"
def test_get_color():
+ assert get_color(10, 10) == "🟢"
assert get_color(0.98) == "🟢"
assert get_color(0.9012) == "🟢"
assert get_color(0.899) == "🟡"
@@ -66,8 +98,12 @@ def test_get_color():
assert get_color(0.0009) == "❌"
assert get_color(0) == "❌"
assert get_color(0, 100) == "❌"
+
+ # Invalid values
assert get_color(-1) == "❌"
assert get_color(None) == "❌"
+ assert get_color("10", 10) == "❌"
+ assert get_color("1") == "❌"
assert get_color("-") == "❌"
assert get_color("test", 100) == "❌"