44f80b5872b1bb9679d15b8230c1731fd26ac527,torchnlp/text_encoders/static_tokenizer_encoder.py,StaticTokenizerEncoder,__init__,#StaticTokenizerEncoder#Any#Any#Any#Any#Any#,15

Before Change

        self.lower = lower
        self.tokenize = tokenize
        self.append_eos = append_eos
        self.tokens = Counter()

        for text in sample:
            self.tokens.update(self._preprocess(text))

        self.stoi = RESERVED_STOI.copy()
        self.itos = RESERVED_ITOS[:]
        for token, count in self.tokens.items():

After Change

        for text in sample:
            self.tokens.update(self.tokenize(text))

        self.stoi = RESERVED_STOI.copy()
        self.itos = RESERVED_ITOS[:]
        for token, count in self.tokens.items():
            if count >= min_occurrences:

Instances