2f85c7fe83c831a41743b4aa67e94f4eb69cdb68,tmtoolkit/preprocess.py,_PreprocWorker,run,#_PreprocWorker#,309

Before Change

    def run(self):
        // print("worker %s running" % self.name)
        while True:
            next_task, task_kwargs = self.tasks_queue.get()
            // print("worker %s got task `%s`" % (self.name, next_task))
            if next_task is None:  // a task of None means shutdown
                self.tasks_queue.task_done()
                break

            exec_task_fn = getattr(self, "_task_" + next_task)
            if exec_task_fn:

After Change

            // print("worker %s has tokens from `%s`" % (self.name, list(self._tokens.keys())))
            self.tasks_queue.task_done()

        self.tasks_queue.task_done()

    def load_tokenizer(self, custom_tokenizer):
        self.tokenizer = custom_tokenizer

Instances