4b21300999e11ba6f91952c05a936ccec0673e2e,nltk/tokenize/treebank.py,TreebankWordTokenizer,span_tokenize,#TreebankWordTokenizer#Any#,147

Before Change

        for word_token in self.tokenize(text):
            if word_token in ("``", """"):
                orig_idx = text.find(word_token, ix)
                quote_idx = text.find(""", ix)
                if orig_idx < 0:
                    real_token = """
                elif quote_idx < 0:
                    real_token = word_token
                elif orig_idx < quote_idx:
                    real_token = word_token
                else:
                    real_token = """
            else:
                real_token = word_token

After Change

        else:
            tokens = raw_tokens

        return align_tokens(tokens, text)


class TreebankWordDetokenizer(TokenizerI):
    

Instances