68b5105ae8c0225f426ddfa8928b9bfe9eb76b76,pycorrector/utils/tokenizer.py,,,#,101

Before Change

           "小牛曲清去蛋白提取物乙"]
    t = Tokenizer()
    for i in txt:
        txt_seg = t.tokenize(i)
        txt_seg_s = t.tokenize(i, "search")
        txt_seg_s_n = t.tokenize(i, "search", False)
        txt_seg_s_f = t.tokenize(i, "default", False)
        print(i)

After Change

if __name__ == "__main__":
    text = "这个消息在北京城里不胫儿走"
    print(text)
    t = Tokenizer()
    print("deault", t.tokenize(text, "default"))
    print("search", t.tokenize(text, "search"))
    print("ngram", t.tokenize(text, "ngram"))

Instances