144aa65949d84526980d53ad1434d0a6ae9cba27,scattertext/TermDocMatrixFactory.py,TermDocMatrixFactory,_get_features_from_parsed_text,#TermDocMatrixFactory#Any#Any#,247

Before Change

	def _get_features_from_parsed_text(self, parsed_text, term_idx_store):
		term_freq = Counter()
		for sent in parsed_text.sents:
			unigrams = []
			for tok in sent:
				if tok.pos_ not in ("PUNCT", "SPACE", "X"):
					if tok.ent_type_ in self._entity_types_to_censor:
						unigrams.append(tok.ent_type_)
					else:
						if self._use_lemmas:
							if tok.lemma_.strip():
								unigrams.append(tok.lemma_.strip())
						else:
							if tok.lower_.strip():
								unigrams.append(tok.lower_.strip())
			bigrams = list(map(" ".join, zip(unigrams[:-1], unigrams[1:])))
			self._augment_term_freq_with_unigrams_and_bigrams(bigrams, term_freq, term_idx_store, unigrams)
		return term_freq

	def _augment_term_freq_with_unigrams_and_bigrams(self, bigrams, term_freq, term_idx_store, unigrams):
		for term in unigrams + bigrams:

After Change

			X_factory[doci, word_idx] = freq

	def _get_features_from_parsed_text(self, parsed_text, term_idx_store):
		return {term_idx_store.getidxstrict(k):v for k, v
		        in self._feats_from_spacy_doc.get_feats(parsed_text).items()
		        if k in term_idx_store}



class FeatsFromDoc(TermDocMatrixFactory):

Instances