44f80b5872b1bb9679d15b8230c1731fd26ac527,torchnlp/text_encoders/static_tokenizer_encoder.py,StaticTokenizerEncoder,__init__,#StaticTokenizerEncoder#Any#Any#Any#Any#Any#,15

Before Change

        self.lower = lower
        self.tokenize = tokenize
        self.append_eos = append_eos
        self.tokens = Counter()

        for text in sample:
            self.tokens.update(self._preprocess(text))

After Change

        if not isinstance(sample, list):
            raise TypeError("Sample needs to be a list of strings.")

        self.append_eos = append_eos
        self.tokens = Counter()
        self.tokenize = tokenize if tokenize else lambda x: x

        for text in sample:
            self.tokens.update(self.tokenize(text))

        self.stoi = RESERVED_STOI.copy()
        self.itos = RESERVED_ITOS[:]
        for token, count in self.tokens.items():
            if count >= min_occurrences:
                self.itos.append(token)
                self.stoi[token] = len(self.itos) - 1

    @property
    def vocab(self):
         Return a list of tokens 
        return self.itos

Instances