994004cca3b3153cc9c5b2d1ff0f6c4ba1b3cd13,tmtoolkit/preprocess/_preprocworker.py,PreprocWorker,_task_lemmatize,#PreprocWorker#,303

Before Change

    def _task_lemmatize(self):
        self._update_docs_attr("text", self._get_docs_attr("lemma_", custom_attr=False))

        if "lemma" in self._std_attrs:
            self._std_attrs.pop(self._std_attrs.index("lemma"))

    def _task_expand_compound_tokens(self, split_chars=("-",), split_on_len=2, split_on_casechange=False):
        exptoks = expand_compounds(self._tokens, split_chars=split_chars, split_on_len=split_on_len,
                                   split_on_casechange=split_on_casechange, flatten=False)

After Change

        assert len(docs_lemmata) == len(self._tokens)
        for doc_tok, doc_lem in zip(self._tokens, docs_lemmata):
            assert len(doc_tok) == len(doc_lem)
            new_docs_lemmata.append([t if l.startswith("-") and l.endswith("-") else l
                                     for t, l in zip(doc_tok, doc_lem)])

        self._update_docs_attr("text", new_docs_lemmata)

Instances