Merge pull request #1806 from flairNLP/GH-1768-tokenizer-default

GH-1768: turn on tokenizer by default in Sentence object
flairNLP · Aug 13, 2020 · 040d386 · 040d386
2 parents e5dde9c + c02aef4
commit 040d386
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/flair/data.py b/flair/data.py
@@ -529,7 +529,7 @@ class Sentence(DataPoint):
     def __init__(
         self,
         text: str = None,
-        use_tokenizer: Union[bool, Tokenizer] = False,
+        use_tokenizer: Union[bool, Tokenizer] = True,
         language_code: str = None,
         start_position: int = None
     ):

diff --git a/tests/test_data.py b/tests/test_data.py
@@ -53,7 +53,7 @@ def test_create_sentence_on_empty_string():
 
 
 def test_create_sentence_without_tokenizer():
-    sentence: Sentence = Sentence("I love Berlin.")
+    sentence: Sentence = Sentence("I love Berlin.", use_tokenizer=False)
 
     assert 3 == len(sentence.tokens)
     assert 0 == sentence.tokens[0].start_pos