37cc2db2c5cfdbb0ea45534259f22e5a36138a45,core/tokenizer.py,,tokenize,#Any#,17

Before Change

    // Protect phrases
    i = 0
    for phrase in protected_phrases:
        if phrase in sentence:
            sentence = sentence.replace(phrase, " PROTECTEDPHRASE{}PROTECTEDPHRASE ".format(i))
        i = i + 1

    // Protected regex-based phrases
    i = 0

After Change

    // Regex-based protected phrases
    protected_phrases_regex_replacements = []
    for i, phrase in enumerate(protected_phrases_regex):

        // If phrase was found in sentence
        if re.search(phrase, sentence):

Instances