4b21300999e11ba6f91952c05a936ccec0673e2e,nltk/tokenize/treebank.py,TreebankWordTokenizer,span_tokenize,#TreebankWordTokenizer#Any#,147

Before Change

            else:
                real_token = word_token
            ix = text.find(real_token, ix)
            end = ix + len(real_token)
            spans.append((ix, end))
            ix = end

        return spans

After Change

        // Do this only if original text contains double quote(s) or double
        // single-quotes (because "" might be transformed to `` if it is
        // treated as starting quotes).
        if (""" in text) or ("""" in text):
            // Find double quotes and converted quotes
            matched = [m.group() for m in re.finditer(r"``|"{2}|\"", text)]
            
            // Replace converted quotes back to double quotes
            tokens = [matched.pop(0) if tok in [""", "``", """"] else tok for tok in raw_tokens]
        else:
            tokens = raw_tokens

        return align_tokens(tokens, text)


class TreebankWordDetokenizer(TokenizerI):

Instances