0e52a77af80ef1aefb4958564d67ffbcdd24cc84,torchnlp/encoders/text/spacy_encoder.py,SpacyEncoder,batch_encode,#SpacyEncoder#Any#,79

Before Change

        super().__init__(*args, tokenize=partial(_tokenize, tokenizer=self.spacy), **kwargs)

    def batch_encode(self, sequences):
        return_ = []
        for tokens in self.spacy.pipe(sequences, n_threads=-1):
            text = [token.text for token in tokens]
            vector = [self.stoi.get(token, self.unknown_index) for token in text]
            if self.append_eos:
                vector.append(self.eos_index)
            return_.append(torch.tensor(vector))
        return return_

After Change

        // Batch tokenization is handled by ``self.spacy.pipe``
        original = self.tokenize
        self.tokenize = lambda sequence: [token.text for token in sequence]
        return_ = super().batch_encode(self.spacy.pipe(sequences))
        self.tokenize = original
        return return_

Instances