0de3c091c75e36ffb4c12a2048110df37cafb644,src/fonduer/candidates/mentions.py,Ngrams,apply,#Ngrams#Any#,58

Before Change

                    and self.split_rgx is not None
                    and end - start > 0
                ):
                    m = re.search(
                        self.split_rgx,
                        context.text[start - offsets[0] : end - offsets[0] + 1],
                    )
                    if m is not None:
                        ts1 = TemporarySpan(
                            char_start=start,
                            char_end=start + m.start(1) - 1,
                            sentence=context,
                        )
                        if ts1 not in seen and ts1.get_span():
                            seen.add(ts1)
                            yield ts1
                        ts2 = TemporarySpan(
                            char_start=start + m.end(1), char_end=end, sentence=context
                        )
                        if ts2 not in seen and ts2.get_span():
                            seen.add(ts2)
                            yield ts2


class MentionNgrams(Ngrams):
    Defines the **space** of Mentions.

    Defines the space of Mentions as all n-grams (n_min <= n <= n_max) in a

After Change

                                seen.add(ts)
                                yield ts
                    // And check splits together
                    for word in re.split("|".join(self.split_tokens), text):
                        ts = TemporarySpan(
                            char_start=text.index(word),
                            char_end=text.index(word) + len(word) - 1,
                            sentence=context,
                        )
                        if ts not in seen and ts.get_span():
                            logger.warning(word)
                            seen.add(ts)
                            yield ts


class MentionNgrams(Ngrams):
    Defines the **space** of Mentions.

    Defines the space of Mentions as all n-grams (n_min <= n <= n_max) in a

Instances