Merge pull request #47 from brunohjs/#33

#33 - Change "response" to "core" and fix somethings in E2E coverage

rasa_model_report/controllers/e2e_coverage_controller.py

@@ -74,7 +74,7 @@ def _generate(self) -> None:
         Generate E2E tests coverage report string.
         """
         not_covered = {}
-        report_data = [item["name"] for item in self.json.responses]
+        report_data = [item["name"] for item in self.json.core]
         for element in ["intents", "entities", "actions"]:
             element_list = getattr(self, f"_{element}")
             not_covered[element] = {

rasa_model_report/controllers/json_controller.py

@@ -33,8 +33,8 @@ def __init__(self, rasa_path: str, output_path: str, project_name: str, project_
         self._entities: List[Dict[str, entity]] = []
         self._entity_overview: Dict[str, float] = {}
         self._entity_errors: List[Dict[str, entity]] = []
-        self._responses: List[Dict[str, float]] = []
-        self._response_overview: Dict[str, float] = {}
+        self._core: List[Dict[str, float]] = []
+        self._core_overview: Dict[str, float] = {}
         self._overview: Dict[str, Union[str, number]] = {
             "project": project_name,
             "version": project_version
@@ -80,7 +80,7 @@ def _load_data(self) -> None:
         self._load_intent_errors()
         self._load_entities()
         self._load_entity_errors()
-        self._load_responses()
+        self._load_core()
         self._load_overview()
 
     def _load_intents(self) -> None:
@@ -134,39 +134,41 @@ def _load_entity_errors(self) -> None:
         """
         self._entity_errors = self.load_json_file(self.entity_errors_path, error_flag=False)
 
-    def _load_responses(self) -> None:
+    def _load_core(self) -> None:
         """
-        Load Rasa response report data.
+        Load Rasa core report data.
         """
-        self._responses = self.load_json_file(self.story_report_path, error_flag=False)
-        if self._responses:
-            self._response_overview = {
-                "macro avg": self._responses.get("macro avg"),
-                "weighted avg": self._responses.get("weighted avg"),
-                "conversation_accuracy": self._responses.get("conversation_accuracy")
+        self._core = self.load_json_file(self.story_report_path, error_flag=False)
+        if self._core:
+            self._core_overview = {
+                "macro avg": self._core.get("macro avg"),
+                "weighted avg": self._core.get("weighted avg"),
+                "conversation_accuracy": self._core.get("conversation_accuracy")
             }
-            del self._responses["macro avg"]
-            del self._responses["weighted avg"]
-            if self._responses.get("accuracy"):
-                del self._responses["accuracy"]
-            if self._responses.get("conversation_accuracy"):
-                del self._responses["conversation_accuracy"]
-            self._responses = self._to_list(self._responses, "f1-score")
+            del self._core["macro avg"]
+            del self._core["weighted avg"]
+            if self._core.get("accuracy"):
+                del self._core["accuracy"]
+            if self._core.get("conversation_accuracy"):
+                del self._core["conversation_accuracy"]
+            self._core = self._to_list(self._core, "f1-score")
 
     def _load_overview(self) -> None:
         """
         Load overview report data.
         """
         intent_overview = self._intent_overview.get("macro avg", {}).get("f1-score")
         entity_overview = self._entity_overview.get("macro avg", {}).get("f1-score")
-        response_overview = self._response_overview.get("macro avg", {}).get("f1-score")
+        core_overview = self._core_overview.get("macro avg", {}).get("f1-score")
         nlu_overview = self._overview.get("nlu")
+        e2e_coverage_overview = self._overview.get("e2e_coverage")
         self._overview.update({
             "updated_at": format_date(),
             "intent": intent_overview if intent_overview is not None else None,
             "entity": entity_overview if entity_overview is not None else None,
-            "response": response_overview if response_overview is not None else None,
-            "nlu": nlu_overview
+            "core": core_overview if core_overview is not None else None,
+            "nlu": nlu_overview,
+            "e2e_coverage": e2e_coverage_overview
         })
         self._calculate_overall()
         if os.path.isfile(self.overview_report_path):
@@ -195,10 +197,11 @@ def save_overview(self) -> None:
 
     def _calculate_overall(self) -> None:
         weights = {
-            "intent": 3,
-            "entity": 2,
-            "response": 1,
-            "nlu": 4
+            "intent": 2,
+            "entity": 1,
+            "core": 1,
+            "nlu": 3,
+            "e2e_coverage": 3
         }
         overview_sum = sum(
             self._overview[item] * w for item, w in weights.items()
@@ -220,6 +223,7 @@ def update_overview(self, obj: Dict[str, number]) -> None:
         :param obj: Object that will be used to update the overview object.
         """
         if isinstance(obj, dict):
+            obj.update({"updated_at": format_date()})
             self._overview.update(obj)
             self._calculate_overall()
 
@@ -278,22 +282,22 @@ def entity_errors(self) -> List[Dict[str, entity]]:
         return self._entity_errors.copy()
 
     @property
-    def responses(self) -> List[Dict[str, float]]:
+    def core(self) -> List[Dict[str, float]]:
         """
-        Get responses data.
+        Get core data.
 
-        :return: Copy of responses data object.
+        :return: Copy of core data object.
         """
-        return self._responses.copy()
+        return self._core.copy()
 
     @property
-    def response_overview(self) -> Dict[str, float]:
+    def core_overview(self) -> Dict[str, float]:
         """
-        Get response overview data.
+        Get core overview data.
 
-        :return: Copy of response overview data object.
+        :return: Copy of core overview data object.
         """
-        return self._response_overview.copy()
+        return self._core_overview.copy()
 
     @property
     def overview(self) -> Dict[str, Union[str, float]]:

rasa_model_report/controllers/markdown_controller.py

@@ -61,7 +61,10 @@ def __init__(
             project_version
         )
 
-        self.json.update_overview({"nlu": self.nlu.general_grade})
+        self.json.update_overview({
+            "nlu": self.nlu.general_grade,
+            "e2e_coverage": self.e2e_coverage.total_rate
+        })
 
     def add_text(self, text: str) -> None:
         """
@@ -122,7 +125,7 @@ def build_summary(self) -> str:
         text += " - [Entities](#entities)\n"
         if self.nlu.is_connected():
             text += " - [NLU](#nlu)\n"
-        text += " - [Responses](#responses)\n"
+        text += " - [Core](#core)\n"
         text += " - [E2E Coverage](#e2e)\n"
         text += "\n"
         return text
@@ -149,7 +152,7 @@ def build_overview(self) -> str:
         :return: Overview report in markdown format.
         """
         overview = self.json.overview
-        for item in ["intent", "entity", "response", "nlu"]:
+        for item in ["intent", "entity", "core", "nlu"]:
             overview[item] = overview[item] if isinstance(overview.get(item), (float, int)) else 0
         text = "## Overview <a name='overview'></a>\n"
         style = "style='font-size:16px'"
@@ -168,17 +171,19 @@ def build_overview(self) -> str:
         text += "|" + "|".join([":-:" for i in range(len(data))]) + "|\n"
         text += "|" + "|".join([f"<span {style}>{item[1]}</span>" for item in data]) + "|\n\n"
         style = "style='font-size:20px'"
-        text += f"|Intent|Entity|NLU|Response|<span {style}>General</span>|\n"
-        text += "|:-:|:-:|:-:|:-:|:-:|\n"
+        text += f"|Intent|Entity|NLU|Core|E2E Coverage|<span {style}>General</span>|\n"
+        text += "|:-:|:-:|:-:|:-:|:-:|:-:|\n"
         text += f"|{change_scale(overview['intent'], 10)}\
             |{change_scale(overview['entity'], 10)}\
             |{change_scale(overview['nlu'], 10)}\
-            |{change_scale(overview['response'], 10)}\
+            |{change_scale(overview['core'], 10)}\
+            |{change_scale(overview['e2e_coverage'], 10)}\
             |<span {style}>**{change_scale(overview['overall'], 10)}**</span>|\n"
         text += f"{get_color(overview['intent'])}\
             |{get_color(overview['entity'])}\
             |{get_color(overview['nlu'])}\
-            |{get_color(overview['response'])}\
+            |{get_color(overview['core'])}\
+            |{get_color(overview['e2e_coverage'])}\
             |<span {style}>{get_color(overview['overall'])}</span>|"
         return text
 
@@ -365,43 +370,43 @@ def build_entity_errors_table(self) -> str:
             text = "\nNo confusions of entities were found in this model.\n"
             return title + text
 
-    def build_response_title(self) -> str:
+    def build_core_title(self) -> str:
         """
-        Build the report response title block.
+        Build the report response and actinos title block.
 
         :return: Title block in markdown format.
         """
-        title = "## Responses <a name='responses'></a>\n"
-        description = "Section that discusses metrics about bot responses and stories.\n"
+        title = "## Core <a name='core'></a>\n"
+        description = "Section that discusses metrics about bot responses and actions.\n"
         return title + description
 
-    def build_response_overview(self) -> str:
+    def build_core_overview(self) -> str:
         """
-        Build the report response overview block.
+        Build the report response and actions overview block.
 
         :return: Overview block in markdown format.
         """
-        response_overview = self.json.response_overview.get("macro avg")
+        core_overview = self.json.core_overview.get("macro avg")
         text = "|Precision|Recall|F1 Score|Examples|\n"
         text += "|:-:|:-:|:-:|:-:|\n"
-        text += f"|{response_overview['precision'] * 100:.1f}%\
-            |{response_overview['recall'] * 100:.1f}%\
-            |{response_overview['f1-score'] * 100:.1f}%\
-            |{response_overview['support']}|\n"
-        text += f"|{get_color(response_overview['precision'])}\
-            |{get_color(response_overview['recall'])}\
-            |{get_color(response_overview['f1-score'])}\
+        text += f"|{core_overview['precision'] * 100:.1f}%\
+            |{core_overview['recall'] * 100:.1f}%\
+            |{core_overview['f1-score'] * 100:.1f}%\
+            |{core_overview['support']}|\n"
+        text += f"|{get_color(core_overview['precision'])}\
+            |{get_color(core_overview['recall'])}\
+            |{get_color(core_overview['f1-score'])}\
             ||\n"
         return text
 
-    def build_response_table(self) -> str:
+    def build_core_table(self) -> str:
         """
-        Função que monta o bloco de texto das histórias no relatório.
+        Build the report core table block.
         """
         title = "### Metrics\n"
-        description = "Table with bot response metrics.\n"
+        description = "Table with bot core metrics.\n"
         title += description + "\n"
-        data = self.json.responses
+        data = self.json.core
         table_data = [[
             "",
             "Response",
@@ -419,7 +424,7 @@ def build_response_table(self) -> str:
             self.csv.save(table_data, "story_report.csv")
             return title + self.build_table(table_data)
         else:
-            text = "\nNo responses were found for this model.\n"
+            text = "\nNo responses or actions were found for this model.\n"
             return title + text
 
     def build_nlu_title(self) -> str:

rasa_model_report/controllers/model_report.py

@@ -93,9 +93,9 @@ def generate_report(self) -> None:
                 self.markdown.add_text(self.markdown.build_nlu_table())
                 self.markdown.add_text(self.markdown.build_nlu_errors_table())
 
-            # Responses
-            self.markdown.add_text(self.markdown.build_response_title())
-            self.markdown.add_text(self.markdown.build_response_table())
+            # Core
+            self.markdown.add_text(self.markdown.build_core_title())
+            self.markdown.add_text(self.markdown.build_core_table())
             self.markdown.add_image(self.dirs['STORY_MATRIX'], "Confusion Matrix")
 
             # E2E Coverage

tests/test_json_controller.py

@@ -79,7 +79,18 @@ def test_load_entities_but_a_non_existent_file():
 def test_load_overview():
     json_controller = pytest.json_controller
     json_controller._load_overview()
-    assert isinstance(json_controller.overview, dict)
+    overview = json_controller.overview
+    assert isinstance(overview, dict)
+    assert isinstance(overview.get("project"), str)
+    assert isinstance(overview.get("version"), str)
+    assert isinstance(overview.get("updated_at"), str)
+    assert isinstance(overview.get("intent"), float)
+    assert isinstance(overview.get("entity"), float)
+    assert isinstance(overview.get("core"), float)
+    assert isinstance(overview.get("nlu"), float) or overview.get("nlu") is None
+    assert isinstance(overview.get("e2e_coverage"), float) or overview.get("e2e_coverage") is None
+    assert isinstance(overview.get("overall"), float)
+    assert isinstance(overview.get("created_at"), str)
 
 
 def test_load_overview_if_exists():
@@ -99,7 +110,7 @@ def test_save_overview():
 def test_calculate_overall():
     json_controller = pytest.json_controller
     json_controller._calculate_overall()
-    assert isinstance(json_controller.overview.get("overall"), (int, float))
+    assert isinstance(json_controller.overview.get("overall"), float)
 
 
 def test_update_overview():
@@ -166,20 +177,20 @@ def test_get_entity_errors():
     assert isinstance(json_controller.entity_errors, list)
 
 
-def test_get_responses():
+def test_get_core():
     json_controller = pytest.json_controller
-    responses = json_controller.responses
-    responses.append({"test": "ok"})
-    assert json_controller.responses != responses
-    assert isinstance(json_controller.responses, list)
+    core = json_controller.core
+    core.append({"test": "ok"})
+    assert json_controller.core != core
+    assert isinstance(json_controller.core, list)
 
 
-def test_get_response_overview():
+def test_get_core_overview():
     json_controller = pytest.json_controller
-    response_overview = json_controller.response_overview
-    response_overview.update({"test": "ok"})
-    assert json_controller.response_overview != response_overview
-    assert isinstance(json_controller.response_overview, dict)
+    core_overview = json_controller.core_overview
+    core_overview.update({"test": "ok"})
+    assert json_controller.core_overview != core_overview
+    assert isinstance(json_controller.core_overview, dict)
 
 
 def test_get_overview():

tests/test_markdown_controller.py

@@ -189,33 +189,33 @@ def test_build_entity_errors_table_if_len_less_than_2():
     assert "No confusions of entities were found in this model" in text
 
 
-def test_build_response_title():
+def test_build_core_title():
     markdown_controller = pytest.markdown_controller
-    text = markdown_controller.build_response_title()
+    text = markdown_controller.build_core_title()
     assert isinstance(text, str)
-    assert "## Responses <a name='responses'></a>" in text
+    assert "## Core <a name='core'></a>" in text
 
 
-def test_build_response_overview():
+def test_build_core_overview():
     markdown_controller = pytest.markdown_controller
-    text = markdown_controller.build_response_overview()
+    text = markdown_controller.build_core_overview()
     assert isinstance(text, str)
 
 
-def test_build_response_table():
+def test_build_core_table():
     markdown_controller = pytest.markdown_controller
-    text = markdown_controller.build_response_table()
+    text = markdown_controller.build_core_table()
     assert isinstance(text, str)
     assert os.path.isfile(f"{markdown_controller.results_path}/story_report.csv")
 
 
-def test_build_response_table_if_len_less_than_2():
+def test_build_core_table_if_len_less_than_2():
     markdown_controller = pytest.markdown_controller
     json_controller = JsonController("invelid/path", "./", "test-project", "0.0.0")
     markdown_controller.json = json_controller
-    text = markdown_controller.build_response_table()
+    text = markdown_controller.build_core_table()
     assert isinstance(text, str)
-    assert "No responses were found for this model" in text
+    assert "No responses or actions were found for this model" in text
     assert not os.path.isfile(f"{markdown_controller.results_path}/story_report.csv")