91184b6bdcdbce3a37cc91a54801b98356910849,theanolm/commands/score.py,,_score_text,#Any#Any#Any#Any#Any#Any#,78

Before Change

                continue
            seq_word_ids = word_ids[:, seq_index]
            output_file.write("////// Sentence {0}\n".format(num_sentences))
            seq_details = [str(word_id) + ":" + str(logprob)
                for word_id, logprob in zip(seq_word_ids, seq_logprobs)]
            output_file.write(" ".join(seq_details) + "\n")
            output_file.write("Sentence perplexity: {0}\n\n".format(
                numpy.exp(-seq_logprob / seq_length)))

After Change

            if not word_level:
                continue
            seq_word_ids = word_ids[:, seq_index]
            seq_class_names = dictionary.ids_to_names(seq_word_ids)
            output_file.write("// Sentence {0}\n".format(num_sentences))
            for word_index, word_logprob in enumerate(seq_logprobs):
                if word_index - 2 > 0:
                    history = seq_class_names[word_index:word_index - 3:-1]
                    history.append("...")
                else:
                    history = seq_class_names[word_index::-1]
                output_file.write("log(p({0} | {1})) = {2}\n".format(
                    seq_class_names[word_index + 1],
                    ", ".join(history),
                    seq_logprobs[word_index]))
            output_file.write("Sentence perplexity: {0}\n\n".format(
                numpy.exp(-seq_logprob / seq_length)))

Instances