2df9a66ac40d1017792b1c93f34e47e214726d7d,demo/load_model/load_and_extract.py,,,#,7

Before Change

        token = line.strip()
        token_dict[token] = len(token_dict)

token_input = np.asarray([[token_dict[token] for token in tokens] + [0] * (512 - len(tokens))])
seg_input = np.asarray([[0] * len(tokens) + [0] * (512 - len(tokens))])

print(token_input[][:len(tokens)])

predicts = model.predict([token_input, seg_input])[0]
for i, token in enumerate(tokens):
    print(token, predicts[i].tolist()[:5])

Official outputs:

After Change

        token = line.strip()
        token_dict[token] = len(token_dict)

tokenizer = Tokenizer(token_dict)
text = "语言模型"
tokens = tokenizer.tokenize(text)
print("Tokens:", tokens)
indices, segments = tokenizer.encode(first="语言模型", max_len=512)

predicts = model.predict([np.array([indices]), np.array([segments])])[0]
for i, token in enumerate(tokens):
    print(token, predicts[i].tolist()[:5])

Instances