1b43ea3d9f7db02075536d9578891af06e324b9a,scattertext/WhitespaceNLP.py,,whitespace_nlp_with_sentences,#Any#Any#Any#,111

Before Change

def whitespace_nlp_with_sentences(doc, entity_type=None, tag_type=None):
	pat = re.compile(r"([A-Z][^\.!?]*[\.!?])", re.M)
	sents = []
	for sent in pat.findall(doc):
		toks = []
		for tok in re.split(r"(\W)", sent):
			pos = "WORD"

After Change

def whitespace_nlp_with_sentences(doc, entity_type=None, tag_type=None):
	pat = re.compile(r"([^\.!?]*?[\.!?$])", re.M)
	sents = []
	raw_sents = pat.findall(doc)
	if len(raw_sents) == 0:
		raw_sents = [doc]
	for sent in raw_sents:
		toks = []
		for tok in re.split(r"(\W)", sent):
			if len(tok) > 0:

Instances