cea292769af4ac688649573a11b20f4d69024e3d,tests/unit_test/processor_units/test_processor_units.py,,test_bert_tokenizer_unit,#,145

Before Change

        "[UNK]", "[CLS]", "[SEP]", "want", "////want", "////ed", "wa", "un", "runn",
        "////ing", ","
    ]
    with tempfile.NamedTemporaryFile(delete=False) as vocab_writer:
        vocab_writer.write("".join(
            [x + "\n" for x in vocab_tokens]).encode("utf-8"))

    vocab_file = vocab_writer.name

    tokenizer_unit = units.BertTokenize(vocab_file, do_lower_case=True)
    os.unlink(vocab_file)

After Change

def test_bert_tokenizer_unit():
    vocab_tokens = [
        "[PAD]", "further", "////more", ",", "under", "the", "micro", "////scope", "neither",
        "entity", "contains", "glands", ".", "此", "外", "在", "显", "微", "镜", "下"
    ]
    raw_text = "furthermore, \r under the microscope \t neither entity  \n contains sebaceous glands. 此外, 在显微镜下"

    golden_tokens = ["further", "////more", ",", "under", "the", "micro", "////scope", "neither", "entity", "contains",
                     "[UNK]", "glands", ".", "此", "外", ",", "在", "显", "微", "镜", "下"]

Instances