1b43ea3d9f7db02075536d9578891af06e324b9a,scattertext/WhitespaceNLP.py,,whitespace_nlp_with_sentences,#Any#Any#Any#,111

Before Change

			pos = "WORD"
			if tok.strip() == "":
				pos = "SPACE"
			elif re.match("^\W+$", tok):
				pos = "PUNCT"
			toks.append(Tok(pos,
			                tok[:2].lower(),

After Change

	pat = re.compile(r"([^\.!?]*?[\.!?$])", re.M)
	sents = []
	raw_sents = pat.findall(doc)
	if len(raw_sents) == 0:
		raw_sents = [doc]
	for sent in raw_sents:
		toks = []
		for tok in re.split(r"(\W)", sent):

Instances