0de3c091c75e36ffb4c12a2048110df37cafb644,src/fonduer/candidates/mentions.py,Ngrams,apply,#Ngrams#Any#,58

Before Change

                            seen.add(ts1)
                            yield ts1
                        ts2 = TemporarySpan(
                            char_start=start + m.end(1), char_end=end, sentence=context
                        )
                        if ts2 not in seen and ts2.get_span():
                            seen.add(ts2)

After Change

                ):
                    text = context.text[start - offsets[0] : end - offsets[0] + 1]
                    // Check splits individually
                    for split_token in self.split_tokens:
                        for word in re.split(split_token, text):
                            ts = TemporarySpan(
                                char_start=text.index(word),
                                char_end=text.index(word) + len(word) - 1,
                                sentence=context,
                            )
                            if ts not in seen and ts.get_span():
                                seen.add(ts)
                                yield ts
                    // And check splits together
                    for word in re.split("|".join(self.split_tokens), text):
                        ts = TemporarySpan(
                            char_start=text.index(word),
                            char_end=text.index(word) + len(word) - 1,

Instances