88e55d70bc603bf54f5d2c64cfb9a0706bafaead,test/data/test_builtin_datasets.py,TestDataset,test_wikitext2,#TestDataset#,46

Before Change

        self.assertEqual(tokens_ids, [2, 286, 503, 700])

        // Add test for the subset of the standard datasets
        train_dataset, test_dataset = torchtext.experimental.datasets.raw.WikiText2(data_select=("train", "test"))
        self._helper_test_func(len(train_dataset), 36718, next(iter(train_dataset)), " \n")
        self._helper_test_func(len(test_dataset), 36718, next(iter(test_dataset)), " \n")
        del train_dataset, test_dataset

After Change

        train_iter, valid_iter, test_iter = torchtext.experimental.datasets.raw.WikiText2(data_select=("train", "valid", "test"))
        self._helper_test_func(len(train_iter), 36718, next(iter(train_iter)), " \n")
        self._helper_test_func(len(valid_iter), 3760, next(iter(valid_iter)), " \n")
        self._helper_test_func(len(test_iter), 4358, next(iter(test_iter)), " \n")
        del train_iter, valid_iter, test_iter
        train_dataset, test_dataset = WikiText2(data_select=("train", "test"))
        self._helper_test_func(len(train_dataset), 2049990, train_dataset[20:25],
                               [5024, 89, 21, 3, 1838])

Instances