286a864220a00732d382a75051e11877acf13c3f,deeppavlov/models/tokenizers/nltk_tokenizer.py,NLTKTokenizer,__init__,#NLTKTokenizer#Any#Any#,24

Before Change

        super().__init__(**kwargs)
        if download:
            nltk.download()
        self.tokenizer = getattr(nltk.tokenize, tokenizer, None)
        if not callable(self.tokenizer):
            raise AttributeError("Tokenizer {} is not defined in nltk.tokenizer".format(tokenizer))

    def infer(self, instance, *args, **kwargs):

After Change

@register("nltk_tokenizer")
class NLTKTokenizer(Inferable):

    def __init__(self, save_path=None, download=False, tokenizer="wordpunct_tokenize", *args, **kwargs):
        super().__init__(save_path=save_path)
        if download:
            nltk.download()
        self.tokenizer = getattr(nltk.tokenize, tokenizer, None)
        if not callable(self.tokenizer):
            raise AttributeError("Tokenizer {} is not defined in nltk.tokenizer".format(tokenizer))

Instances