if line_as_list[0] in vocab:
dictionary[line_as_list[0]] = numpy.array(
[float(num) for num in line_as_list[-300:]])
vocab.remove(line_as_list[0])
del vocab
create_benchmark(args.input, dictionary)
After Change
with open(split_input_fst, "r") as f:
for line in f:
for word in tokenize_sentence(line.strip().split()):
vocab[word] = vocab.get(word, 0) + 1
word_count += 1
with open(split_input_snd, "r") as f:
for line in f:
for word in tokenize_sentence(line.strip().split()):
vocab[word] = vocab.get(word, 0) + 1
word_count += 1
for word in vocab:
vocab[word] /= word_count
dictionary = {}
with open(args.vocab, "r") as v:
for line in v:
