2573c649518391ada6214cfc72d20421dfac4072,src/preprocess.py,,get_embeddings,#Any#Any#Any#,194

Before Change

    embeddings = np.zeros((word_v_size, d_word))
    for idx in range(word_v_size): // kind of hacky
        word = vocab.get_token_from_index(idx)
        if word == "@@PADDING@@" or word == "@@UNKNOWN@@":
            continue
        try:
            assert word in word2vec
        except AssertionError as error:
            log.debug(error)

After Change

    word_v_size, unk_idx = vocab.get_vocab_size("tokens"), vocab.get_token_index(vocab._oov_token)
    embeddings = np.random.randn(word_v_size, d_word) //np.zeros((word_v_size, d_word))
    with open(vec_file) as vec_fh:
        for line in vec_fh:
            word, vec = line.split(" ", 1)
            idx = vocab.get_token_index(word)
            if idx != unk_idx:
                idx = vocab.get_token_index(word)
                embeddings[idx] = np.array(list(map(float, vec.split())))
    embeddings[vocab.get_token_index("@@PADDING@@")] = 0.
    embeddings = torch.FloatTensor(embeddings)
    log.info("\tFinished loading embeddings")
    return embeddings

Instances