9f2c085d688ebc0e26f00a3ddd5d7b301ce2ff0f,pytext/utils/data.py,,unkify,#Any#,244

Before Change

def unkify(token):  // noqa: C901
    if any(char.isdigit() for char in token.rstrip()):
        return VocabMeta.UNK_NUM_TOKEN

    return VocabMeta.UNK_TOKEN

After Change

@torch.jit.script
def unkify(token: str):
    res = "<unk>"
    for idx in range(len(token)):
        if token[idx].isdigit():
            res = "<unk>-NUM"

    return res

Instances