ce8df6284dfc00952bc43dfb606a858dbc2639a9,pytext/data/test/query_document_pairwise_ranking_data_handler_test.py,QueryDocumentPairwiseRankingDataHandlerTest,test_tokenization,#QueryDocumentPairwiseRankingDataHandlerTest#,53

Before Change

    def test_tokenization(self):
        file_name = tests_module.test_file("query_document_pairwise_ranking_tiny.tsv")
        data = self.data_handler.read_from_file(
            file_name, self.data_handler.raw_columns
        )
        data = list(self.data_handler.preprocess(data))
        print(data[0][ModelInput.QUERY])
        // test tokenization: must split into chars (ascii string input)
        self.assertEqual(data[0][ModelInput.QUERY], ["q", "u", "e", "r", "y"])

After Change

    def test_tokenization(self):
        file_name = tests_module.test_file("query_document_pairwise_ranking_tiny.tsv")
        data = list(
            self.data_handler.read_from_file(file_name, self.data_handler.raw_columns)
        )
        data = list(self.data_handler.preprocess(data))
        print(data[0][ModelInput.QUERY])
        // test tokenization: must split into chars (ascii string input)
        self.assertEqual(data[0][ModelInput.QUERY], ["q", "u", "e", "r", "y"])

Instances