2df9a66ac40d1017792b1c93f34e47e214726d7d,demo/load_model/load_and_pool.py,,,#,9

Before Change

        token = line.strip()
        token_dict[token] = len(token_dict)

token_input = np.asarray([[token_dict[token] for token in tokens] + [0] * (512 - len(tokens))])
seg_input = np.asarray([[0] * len(tokens) + [0] * (512 - len(tokens))])

print("Inputs:", token_input[0][:len(tokens)])

After Change

tokenizer = Tokenizer(token_dict)
text = "语言模型"
tokens = tokenizer.tokenize(text)
print("Tokens:", tokens)
indices, segments = tokenizer.encode(first="语言模型", max_len=512)

predicts = model.predict([np.array([indices]), np.array([segments])])[0]
print("Pooled:", predicts.tolist()[:5])

Instances