diff --git a/medcat/utils/regression/regression_checker.py b/medcat/utils/regression/regression_checker.py index 6dce35d5..c2bff539 100644 --- a/medcat/utils/regression/regression_checker.py +++ b/medcat/utils/regression/regression_checker.py @@ -38,7 +38,8 @@ def main(model_pack_dir: Path, test_suite_file: Path, use_mct_export: bool = False, mct_export_yaml_path: Optional[str] = None, only_mct_export_conversion: bool = False, - only_describe: bool = False) -> None: + only_describe: bool = False, + require_fully_correct: bool = False) -> None: """Check test suite against the specifeid model pack. Args: @@ -60,6 +61,9 @@ def main(model_pack_dir: Path, test_suite_file: Path, I.e exit when MCT export conversion is done. Defaults to False. only_describe (bool): Whether to only describe the finding options and exit. Defaults to False. + require_fully_correct (bool): Whether all cases are required to be correct. + If set to True, an exit-status of 1 is returned unless all (sub)cases are correct. + Defaults to False. Raises: ValueError: If unable to overwrite file or folder does not exist. @@ -103,6 +107,12 @@ def main(model_pack_dir: Path, test_suite_file: Path, logger.info(res.get_report(phrases_separately=phrases, hide_empty=hide_empty, examples_strictness=examples_strictness, strictness=strictness, phrase_max_len=max_phrase_length)) + if require_fully_correct: + total, success = res.calculate_report(phrases_separately=phrases, + hide_empty=hide_empty, examples_strictness=examples_strictness, + strictness=strictness, phrase_max_len=max_phrase_length)[:2] + if total != success: + exit(1) if __name__ == '__main__': @@ -147,6 +157,10 @@ def main(model_pack_dir: Path, test_suite_file: Path, action='store_true') parser.add_argument('--only-describe', help='Only describe the various findings and exit.', action='store_true') + parser.add_argument('--require-fully-correct', help='Require the regression test to be fully correct. ' + 'If set, a non-zero exit status is returned unless all cases are successfull (100%). ' + 'This can be useful for (e.g) CI workflow integration.', + action='store_true') args = parser.parse_args() if not args.silent: logger.addHandler(logging.StreamHandler()) @@ -160,4 +174,5 @@ def main(model_pack_dir: Path, test_suite_file: Path, jsonpath=args.jsonfile, overwrite=args.overwrite, jsonindent=args.jsonindent, strictness_str=args.strictness, max_phrase_length=args.max_phrase_length, use_mct_export=args.from_mct_export, mct_export_yaml_path=args.mct_export_yaml, - only_mct_export_conversion=args.only_conversion, only_describe=args.only_describe) + only_mct_export_conversion=args.only_conversion, only_describe=args.only_describe, + require_fully_correct=args.require_fully_correct)