// number of words, including <unk>"s
num_words += len(merged_words)
// number of word probabilities computed (may not include <unk>"s)
num_probs += len(filter(None, merged_logprobs))
// number of <unk>"s (just for reporting)
num_unks += merged_words.count("<unk>")
// number of sequences
num_sentences += 1
After Change
// number of words, including <unk>"s
num_words += len(merged_words)
// number of word probabilities computed (may not include <unk>"s)
num_probs += sum(not x is None for x in merged_logprobs)
// number of <unk>"s (just for reporting)
num_unks += sum(x is None for x in merged_logprobs)
// number of sequences
num_sentences += 1
