eee9f1bbdaa88fa4fde7c3c8096a656804baf9e1,tests/test_tokenizers.py,TestTokenizer,test_tokenize_sentence,#TestTokenizer#,19

Before Change

        tokenizer = Tokenizer("english")
        words = tokenizer.to_words("I am a very nice sentence with comma, but..")

        expected = [
            "I", "am", "a", "very", "nice", "sentence",
            "with", "comma", ",", "but.."
        ]
        self.assertEqual(expected, words)

    def test_tokenize_paragraph(self):
        tokenizer = Tokenizer("english")

After Change

        tokenizer = Tokenizer("english")
        words = tokenizer.to_words("I am a very nice sentence with comma, but..")

        expected = (
            "I", "am", "a", "very", "nice", "sentence",
            "with", "comma",
        )
        self.assertEqual(expected, words)

    def test_tokenize_paragraph(self):
        tokenizer = Tokenizer("english")

Instances