8c4dd89b40febc9d2493b7410aac530b6968ce70,gluonnlp/data/stream.py,_LanguageModelBPTTStream,__iter__,#_LanguageModelBPTTStream#,418

Before Change

        data = np.empty([self._batch_size, self._seq_len], dtype=np.float32)
        target = np.empty([self._batch_size, self._seq_len], dtype=np.float32)
        mask = np.empty([self._batch_size, self._seq_len], dtype=np.float32)
        corpus = iter(self._corpus)

        while has_next or has_token_buffered:
            _init(data, target, mask, self._padding_idx)
            has_token_buffered = False

After Change

        target = np.empty([self._batch_size, self._seq_len], dtype=np.float32)
        mask = np.empty([self._batch_size, self._seq_len], dtype=np.float32)
        corpus = itertools.chain.from_iterable(
            (corpus_dataset[idx] for idx in self._sampler(len(corpus_dataset)))
            for corpus_dataset in self._corpus)

        while has_next or has_token_buffered:
            _init(data, target, mask, self._padding_idx)

Instances