68b5105ae8c0225f426ddfa8928b9bfe9eb76b76,pycorrector/utils/tokenizer.py,,,#,101

Before Change

if __name__ == "__main__":
    txt = ["我不要你花钱,这些路曲近通幽",
           "这个消息不胫儿走",
           "这个消息不径而走",
           "这个消息不胫而走",
           "复方甘草口服溶液限田基",
           "张老师经常背课到深夜，我们要体晾老师的心苦。",
           "新进人员时，知识当然还不过，可是人有很有精神，面对工作很认真的话，很快就学会、体会。",
           "小牛曲清去蛋白提取物乙"]
    t = Tokenizer()
    for i in txt:
        txt_seg = t.tokenize(i)
        txt_seg_s = t.tokenize(i, "search")
        txt_seg_s_n = t.tokenize(i, "search", False)
        txt_seg_s_f = t.tokenize(i, "default", False)
        print(i)
        print(txt_seg)
        print(txt_seg_s)
        print(txt_seg_s_n)
        print(txt_seg_s_f)

After Change

    text = "这个消息在北京城里不胫儿走"
    print(text)
    t = Tokenizer()
    print("deault", t.tokenize(text, "default"))
    print("search", t.tokenize(text, "search"))
    print("ngram", t.tokenize(text, "ngram"))

Instances