d264e82050700d9aaed31c11dbd65f9dbd03e4d9,snips_nlu/tokenization.py,,tokenize,#Any#,11

Before Change

def tokenize(string):
    return [Token(m.group(), m.start(), m.end())
            for m in TOKEN_REGEX.finditer(string)]


def tokenize_light(string):

After Change

def tokenize(string):
    return _tokenize(string, [WORD_REGEX, SYMBOL_REGEX])


def _tokenize(string, regexes):
    non_overlapping_tokens = []

Instances