2c4a6e537126f4123de7c97f30587310d3712c06,tests/data/tokenizers/word_splitter_test.py,TestSimpleWordSplitter,test_tokenize_handles_special_cases,#TestSimpleWordSplitter#,39

Before Change

        sentence = "mr. and mrs. jones, etc., went to, e.g., the store"
        expected_tokens = ["mr.", "and", "mrs.", "jones", ",", "etc.", ",", "went", "to", ",",
                           "e.g.", ",", "the", "store"]
        tokens, _ = self.word_splitter.split_words(sentence)
        assert tokens == expected_tokens


class TestSpacyWordSplitter(AllenNlpTestCase):

After Change

        sentence = "mr. and mrs. jones, etc., went to, e.g., the store"
        expected_tokens = ["mr.", "and", "mrs.", "jones", ",", "etc.", ",", "went", "to", ",",
                           "e.g.", ",", "the", "store"]
        tokens = [t.text for t in self.word_splitter.split_words(sentence)]
        assert tokens == expected_tokens


class TestSpacyWordSplitter(AllenNlpTestCase):

Instances