0f95d969bb87f8e6abd33473edc9a58a345dc9d4,snips_nlu/tests/test_preprocessing.py,TestPreprocessing,test_should_tokenize_symbols,#TestPreprocessing#,50

Before Change

        tokens = tokenize(text, language)

        // Then
        expected_tokens = [
            Token(value="$$", start=0, end=2),
            Token(value="%", start=3, end=4),
            Token(value="!!", start=5, end=7)
        ]
        self.assertListEqual(tokens, expected_tokens)

    def test_space_should_by_ignored(self):
        // Given

After Change

        tokens = tokenize(text, language)

        // Then
        expected_tokens = [
            Token(value="$", start=0, end=1),
            Token(value="$", start=1, end=2),
            Token(value="%", start=3, end=4),
            Token(value="!", start=5, end=6),
            Token(value="!", start=6, end=7)
        ]
        self.assertListEqual(tokens, expected_tokens)

    def test_space_should_by_ignored(self):
        // Given

Instances