8de95b27c53cc83b249a18c4b36bc4d82524cace,tmtoolkit/preprocess/utils.py,,remove_special_chars_in_tokens,#Any#Any#,87

Before Change

    if not special_chars:
        raise ValueError("`special_chars` must be a non-empty sequence")

    special_chars_str = u"".join(special_chars)

    if "maketrans" in dir(string):  // python 2
        del_chars = {ord(c): None for c in special_chars}
        return [t.translate(del_chars) for t in tokens]
    elif "maketrans" in dir(str):  // python 3
        del_chars = str.maketrans("", "", special_chars_str)
        return [t.translate(del_chars) for t in tokens]
    else:
        raise RuntimeError("no maketrans() function found")


def create_ngrams(tokens, n, join=True, join_str=" "):
    if n < 2:
        raise ValueError("`n` must be at least 2")

After Change

@deprecated(deprecated_in="0.9.0", removed_in="0.10.0", details="Method was renamed to `remove_chars_in_tokens`.")
def remove_special_chars_in_tokens(tokens, special_chars):
    return remove_chars_in_tokens(tokens, special_chars)


def remove_chars_in_tokens(tokens, chars):
    if not chars:

Instances