eba7183bd6fd90dd15c5daa260a836c64682018c,pytext/data/bert_tensorizer.py,BERTTensorizerBase,numberize,#BERTTensorizerBase#Any#,338

Before Change

        
        sentences = [self._lookup_tokens(row[column])[0] for column in self.columns]
        sentences = self._wrap_numberized_text(sentences)
        seq_lens = (len(sentence) for sentence in sentences)
        segment_labels = ([i] * seq_len for i, seq_len in enumerate(seq_lens))
        tokens = list(itertools.chain(*sentences))
        segment_labels = list(itertools.chain(*segment_labels))
        seq_len = len(tokens)
        positions = list(range(seq_len))
        // tokens, segment_label, seq_len
        return tokens, segment_labels, seq_len, positions

    def tensorize(self, batch) -> Tuple[torch.Tensor, ...]:
        
        Convert instance level vectors into batch level tensors.

After Change

        per_sentence_tokens = [
            self.tokenizer.tokenize(row[column]) for column in self.columns
        ]
        return self.tensorizer_script_impl.numberize(per_sentence_tokens)

    def tensorize(self, batch) -> Tuple[torch.Tensor, ...]:
        
        Convert instance level vectors into batch level tensors.

Instances