1b43ea3d9f7db02075536d9578891af06e324b9a,scattertext/WhitespaceNLP.py,,whitespace_nlp_with_sentences,#Any#Any#Any#,111

Before Change

		toks = []
		for tok in re.split(r"(\W)", sent):
			pos = "WORD"
			if tok.strip() == "":
				pos = "SPACE"
			elif re.match("^\W+$", tok):
				pos = "PUNCT"
			toks.append(Tok(pos,
			                tok[:2].lower(),
			                tok.lower(),
			                ent_type="" if entity_type is None else entity_type.get(tok, ""),

After Change

	pat = re.compile(r"([^\.!?]*?[\.!?$])", re.M)
	sents = []
	raw_sents = pat.findall(doc)
	if len(raw_sents) == 0:
		raw_sents = [doc]
	for sent in raw_sents:
		toks = []
		for tok in re.split(r"(\W)", sent):
			if len(tok) > 0:

Instances