959fcc4f5dd2972a1894041a3aff1505f61fcf93,tmtoolkit/preprocess.py,TMPreproc,tokenize,#TMPreproc#,154

Before Change

        return self

    def tokenize(self):
        if not callable(self.tokenizer):
            raise ValueError("tokenizer must be callable")

        self._tokens = {dl: tuplize(self.tokenizer(txt)) for dl, txt in self.docs.items()}

        return self

After Change

    def tokenize(self):
        self._invalidate_workers_tokens()
        self._send_task_to_workers("tokenize")
        return self

    def generate_ngrams(self, n, join=True, join_str=" ", reassign_tokens=False):
        //self._require_tokens()

Instances