8771290e987cd693bc38ad9339ac4cc6a70d6d2c,pytext/data/test/tensorizers_test.py,TensorizersTest,test_create_byte_tensors,#TensorizersTest#,73

Before Change

        chars, seq_lens = tensorizer.create_training_tensors(batch)
        self.assertIsInstance(chars, torch.LongTensor)
        self.assertIsInstance(seq_lens, torch.LongTensor)
        self.assertEqual((2, max(len(s1), len(s2))), chars.size())
        self.assertEqual((2,), seq_lens.size())
        self.assertEqual(expected, chars.tolist())
        self.assertEqual([len(s1), len(s2)], seq_lens.tolist())

After Change

        rows = [{"text": types.Text(s1)}, {"text": types.Text(s2)}]
        expected = [[ord(c) for c in s1], [ord(c) for c in s2]]

        tensors = (tensorizer.numberize(row) for row in rows)
        chars, seq_len = next(tensors)
        self.assertEqual(len(s1), len(chars))
        self.assertEqual(expected[0], chars)
        self.assertEqual(len(s1), seq_len)

        chars, seq_len = next(tensors)
        self.assertEqual(len(s2), len(chars))
        self.assertEqual(expected[1], chars)
        self.assertEqual(len(s2), seq_len)

Instances