2f85c7fe83c831a41743b4aa67e94f4eb69cdb68,tmtoolkit/preprocess.py,_PreprocWorker,run,#_PreprocWorker#,309

Before Change

    def run(self):
        // print("worker %s running" % self.name)
        while True:
            next_task, task_kwargs = self.tasks_queue.get()
            // print("worker %s got task `%s`" % (self.name, next_task))
            if next_task is None:  // a task of None means shutdown
                self.tasks_queue.task_done()
                break

After Change

    def run(self):
        // print("worker %s running" % self.name)
        for next_task, task_kwargs in iter(self.tasks_queue.get, None):
            //next_task, task_kwargs = self.tasks_queue.get()
            // print("worker %s got task `%s`" % (self.name, next_task))
            // if next_task is None:  // a task of None means shutdown
            //     self.tasks_queue.task_done()
            //     break

            exec_task_fn = getattr(self, "_task_" + next_task)
            if exec_task_fn:
                exec_task_fn(**task_kwargs)
            else:
                raise NotImplementedError("Task not implemented: `%s`" % next_task)

            // print("worker %s has tokens from `%s`" % (self.name, list(self._tokens.keys())))
            self.tasks_queue.task_done()

        self.tasks_queue.task_done()

    def load_tokenizer(self, custom_tokenizer):
        self.tokenizer = custom_tokenizer

Instances