1b68a9d690f164190075fd5cf16937d191dafe75,pytext/data/bert_tensorizer.py,BERTTensorizerBase,tensorize,#BERTTensorizerBase#Any#,304

Before Change

        
        Convert instance level vectors into batch level tensors.
        
        return self.tensorizer_script_impl.tensorize_wrapper(*zip(*batch))

    def initialize(self, vocab_builder=None, from_scratch=True):
        // vocab for BERT is already set
        return

After Change

        
        tokens, segment_labels, seq_lens, positions = zip(*batch)
        tokens = pad_and_tensorize(tokens, self.vocab.get_pad_index())
        pad_mask = (tokens != self.vocab.get_pad_index()).long()
        segment_labels = pad_and_tensorize(segment_labels)
        positions = pad_and_tensorize(positions)
        return tokens, pad_mask, segment_labels, positions

    def initialize(self, vocab_builder=None, from_scratch=True):
        // vocab for BERT is already set
        return

Instances