assert "_STR:0_" in primitive_vocab
// generate vocabulary for the code tokens!
code_tokens = [tokenize_code(e["tgt_canonical_code"], mode="decoder")for e in loaded_examples]
code_vocab = VocabEntry.from_corpus(code_tokens, size=5000, freq_cutoff=vocab_freq_cutoff)
vocab = Vocab(source=src_vocab, primitive=primitive_vocab, code=code_vocab)
After Change
assert "_STR:0_" in primitive_vocab
// generate vocabulary for the code tokens!
code_tokens = [tokenize_code(e.tgt_code, mode="decoder") for e in train_examples]
code_vocab = VocabEntry.from_corpus(code_tokens, size=5000, freq_cutoff=vocab_freq_cutoff)
vocab = Vocab(source=src_vocab, primitive=primitive_vocab, code=code_vocab)
