f4af38defd9b7c1a478f7336b8754020155f9147,torchtext/experimental/datasets/language_modeling.py,,_setup_datasets,#Any#Any#Any#Any#Any#Any#Any#Any#,64

Before Change

            raw_data[name], = raw.DATASETS[dataset_name](root=root, data_select=name, year=year, language=language)
        else:
            raw_data[name], = raw.DATASETS[dataset_name](root=root, data_select=name)
        raw_data[name] = [text_transform(txt) for txt in raw_data[name]]

    return tuple(LanguageModelingDataset(raw_data[item], vocab, text_transform, single_line)
                 for item in data_select)

After Change

        raw_datasets = raw.DATASETS[dataset_name](root=root, data_select=data_select, year=year, language=language)
    else:
        raw_datasets = raw.DATASETS[dataset_name](root=root, data_select=data_select)
    raw_data = {name: list(map(text_transform, raw_dataset)) for name, raw_dataset in zip(data_select, raw_datasets)}

    return tuple(LanguageModelingDataset(raw_data[item], vocab, text_transform, single_line)
                 for item in data_select)

Instances