cea292769af4ac688649573a11b20f4d69024e3d,tests/unit_test/processor_units/test_processor_units.py,,test_bert_tokenizer_unit,#,145

Before Change

    ]
    with tempfile.NamedTemporaryFile(delete=False) as vocab_writer:
        vocab_writer.write("".join(
            [x + "\n" for x in vocab_tokens]).encode("utf-8"))

    vocab_file = vocab_writer.name

After Change

    clean_unit = units.BertClean()
    cleaned_text = clean_unit.transform(raw_text)
    chinese_tokenize_unit = units.ChineseTokenize()
    chinese_tokenized_text = chinese_tokenize_unit.transform(cleaned_text)
    basic_tokenize_unit = units.BasicTokenize()
    basic_tokens = basic_tokenize_unit.transform(chinese_tokenized_text)
    wordpiece_unit = units.WordPieceTokenize(vocab_dict)
    wordpiece_tokens = wordpiece_unit.transform(basic_tokens)

Instances