c52f6ea4fc8f58b389fd806c117f501fb8234976,fairseq/data.py,,_make_batches,#Any#Any#Any#Any#Any#Any#Any#Any#,293

Before Change

        sample_len = max(sample_len, src_size, dst_size)
        num_tokens = (len(batch) + 1) * sample_len
        if yield_batch(idx, num_tokens):
            yield batch
            batch = []
            sample_len = max(src_size, dst_size)

        batch.append(idx)

After Change

        sample_len = max(sample_len, src_size, dst_size)
        num_tokens = (len(batch) + 1) * sample_len
        while yield_batch(idx, num_tokens):
            mod8_len = max(8 * (len(batch) // 8), len(batch) % 8)
            yield batch[:mod8_len]
            batch = batch[mod8_len:]
            sample_len = max([max(src.sizes[id], dst.sizes[id]) for id in batch]) if len(batch) > 0 else 0
            sample_len = max(sample_len, src_size, dst_size)
            num_tokens = (len(batch) + 1) * sample_len

        batch.append(idx)

    if len(batch) > 0:

Instances