387e2c43970dcb6a5f57fda43662b3e9a8df0674,tmtoolkit/preprocess/_preprocworker.py,PreprocWorker,_task_transform_tokens,#PreprocWorker#Any#,288

Before Change

        self._ngrams = {}

    def _task_transform_tokens(self, transform_fn, **kwargs):
        self._update_docs_tokenattrs("text", transform(self._tokens, transform_fn, **kwargs))

    def _task_tokens_to_lowercase(self):
        self._update_docs_tokenattrs("text", _get_docs_tokenattrs(self._docs, "lower_", custom_attr=False))

After Change

        self._ngrams = {}

    def _task_transform_tokens(self, transform_fn, **kwargs):
        for doc, new_tok in zip(self._docs, transform(self._tokens, transform_fn, **kwargs)):
            doc.user_data["tokens"] = new_tok

    def _task_tokens_to_lowercase(self):
        for doc, new_tok in zip(self._docs, _get_docs_tokenattrs(self._docs, "lower_", custom_attr=False)):
            doc.user_data["tokens"] = new_tok

Instances