22a7e3b66d306446852063a0a5320a30667c57dd,tmtoolkit/preprocess/_tmpreproc.py,TMPreproc,remove_tokens_by_doc_frequency,#TMPreproc#Any#Any#Any#,1300

Before Change

        if blacklist:
            self._invalidate_workers_tokens()

            logger.debug("will remove the following %d tokens: %s" % (len(blacklist), blacklist))
            self._send_task_to_workers("filter_tokens",
                                       search_tokens=blacklist,
                                       match_type="exact",
                                       ignore_case=False,

After Change

                                               df_threshold=df_threshold, absolute=absolute, return_mask=True)

        if sum(sum(dmsk) for dmsk in mask) > 0:
            self.remove_tokens_by_mask(dict(zip(self._workers_tokens.keys(), mask)))

        return self

Instances