32335404f09c47cccbfbf731abc4c510d0eef043,fairseq/data/concat_dataset.py,ConcatDataset,sizes,#ConcatDataset#,72

Before Change

    @property
    def sizes(self):
        return np.concatenate(
            [np.tile(ds.sizes, sr) for ds, sr in zip(self.datasets, self.sample_ratios)]
        )

    @property
    def supports_prefetch(self):
        return all(d.supports_prefetch for d in self.datasets)

After Change

    def sizes(self):
        _dataset_sizes = []
        for ds, sr in zip(self.datasets, self.sample_ratios):
            if isinstance(ds.sizes, np.ndarray):
                _dataset_sizes.append(np.tile(ds.sizes, sr))
            else:
                // Only support underlying dataset with single size array.
                assert isinstance(ds.sizes, list)
                _dataset_sizes.append(np.tile(ds.sizes[0], sr))
        return np.concatenate(_dataset_sizes)

    @property
    def supports_prefetch(self):

Instances