eee9f1bbdaa88fa4fde7c3c8096a656804baf9e1,sumy/nlp/tokenizers.py,Tokenizer,to_words,#Tokenizer#Any#,34

Before Change

        return tuple(map(unicode.strip, sentences))

    def to_words(self, sentence):
        return nltk.word_tokenize(to_unicode(sentence))

After Change

        return tuple(map(unicode.strip, sentences))

    def to_words(self, sentence):
        words = nltk.word_tokenize(to_unicode(sentence))
        return tuple(filter(self._is_word, words))

    def _is_word(self, word):
        return bool(Tokenizer._WORD_PATTERN.search(word))

Instances