Handle zero-width space (#18)

Split on zero width whitespace character U+200B

syntok/tokenizer.py

@@ -94,8 +94,9 @@ class Tokenizer:
     )
     """Secondary regex to sub-split non-whitespace sequences."""
 
-    _spaces = regex.compile(r"\S+", regex.UNICODE)
-    """Primary regex to split strings at any kind of Unicode whitespace."""
+    # Annoyingly, unicode regex character class \S does not include the zwsp...
+    _spaces = regex.compile(r"[^\s\u200b]+", regex.UNICODE)
+    """Primary regex to split strings at any kind of Unicode whitespace and the zero width space (zwsp)."""
 
     @staticmethod
     def join_hyphenated_words_across_linebreaks(text: str) -> str:

syntok/tokenizer_test.py

@@ -69,7 +69,7 @@ def test_emit_underscore(self):
         self.assertListEqual(s(self.tokenizer.split("ab_cd")), ["ab", "_", "cd"])
 
     def test_spacing_prefix(self):
-        text = " Hi man,  spaces !! "
+        text = " Hi man,  spaces of \u200Ball  kinds!! "
         output = self.tokenizer.split(text)
         reconstruction = "".join(map(str, output))
         self.assertEqual(text, reconstruction)

syntok/tokenizer_test.txt

@@ -38,6 +38,8 @@ Weird -usage- of hyphens.
 Weird - usage - of hyphens .
 Split hidden,sentence terminal.Here the user missed,a space.
 Split hidden , sentence terminal . Here the user missed , a space .
+Split zero width​space characters.
+Split zero width space characters .
 Special a,b a:a b;b a.b A.B and Aa.B and A!B Aa!B and A?A Aa?B
 Special a , b a:a b ; b a.b A.B and Aa . B and A!B Aa ! B and A?A Aa ? B
 Srᵃ “A&D” yes-no CaMel U.S.A. *@#$^%! $10.00 Nº (one_two): a@b.c is 1ᵃ.