90e5b974a7173ab0bba0990a690e32f25f5b725a,tools/embeddings_to_torch.py,,get_vocabs,#Any#,13

Before Change

def get_vocabs(dict_file):
    vocabs = torch.load(dict_file)

    enc_vocab, dec_vocab = None, None

    // the vocab object is a list of tuple (name, torchtext.Vocab)
    // we iterate over this list and associate vocabularies based on the name

After Change

def get_vocabs(dict_path):
    fields = torch.load(dict_path)

    if old_style_vocab(fields):
        enc_vocab = next((v for n, v in fields if n == "src"), None)
        dec_vocab = next((v for n, v in fields if n == "tgt"), None)
    else:
        enc_vocab = fields["src"][0][1].vocab
        dec_vocab = fields["tgt"][0][1].vocab

    logger.info("From: %s" % dict_path)
    logger.info("\t* source vocab: %d words" % len(enc_vocab))
    logger.info("\t* target vocab: %d words" % len(dec_vocab))

Instances