7c90da5ceb76241d516957f142507fccc51b7081,pythainlp/tokenize/__init__.py,,sent_tokenize,#Any#Any#,95

Before Change

    sentences = []

    if engine == "whitespace":
        sentences = nltk.tokenize.WhitespaceTokenizer().tokenize(text)
    else:  // default, use whitespace + newline
        sentences = re.sub(r"\n+|\s+", "|", text.strip()).split("|")

    return sentences

After Change

    if engine == "whitespace":
        sentences = re.split(r" +", text, re.U)
    else:  // default, use whitespace + newline
        sentences = text.split()

    return sentences

Instances