0bcc8da0344cddc9dfff82a788df519c19489500,nltk/tokenize/treebank.py,TreebankWordTokenizer,span_tokenize,#TreebankWordTokenizer#Any#,147

Before Change

            True

        
        raw_tokens = self.tokenize(text)

        // Convert converted quotes back to original double quotes
        // Do this only if original text contains double quote(s)
        if """ in text:
            // Find double quotes and converted quotes
            matched = [m.group() for m in re.finditer(r"[(``)(\"\")(")]+", text)]
            
            // Replace converted quotes back to double quotes
            tokens = [matched.pop(0) if tok in [""", "``", """"] else tok for tok in raw_tokens]
        else:
            tokens = raw_tokens

        return align_tokens(tokens, text)

After Change

        // Convert converted quotes back to original double quotes
        ix = 0

        spans = []
        for word_token in self.tokenize(text):
            if word_token in ("``", """"):
                orig_idx = text.find(word_token, ix)
                quote_idx = text.find(""", ix)
                if orig_idx < 0:
                    real_token = """
                elif quote_idx < 0:
                    real_token = word_token
                elif orig_idx < quote_idx:
                    real_token = word_token
                else:
                    real_token = """
            else:
                real_token = word_token
            ix = text.find(real_token, ix)
            end = ix + len(real_token)
            spans.append((ix, end))
            ix = end

        return spans


class TreebankWordDetokenizer(TokenizerI):

Instances