b73fc0c21ea55c7d3357fbf633944e1e67db8bdf,theanolm/scoring/latticedecoder.py,LatticeDecoder,decode,#LatticeDecoder#Any#,199

Before Change

            node_tokens = tokens[node.id]
            assert node_tokens
            if node.id == lattice.final_node.id:
                new_tokens = [self.Token.copy(token) for token in node_tokens]
                self.append_word(new_tokens, self._eos_id, lm_scale, wi_penalty)
                return sorted(new_tokens,
                              key=lambda token: token.total_logprob,
                              reverse=True)

After Change

                tokens[link.end_node.id][100:] = []

            nodes_processed += 1
            if nodes_processed % math.ceil(len(sorted_nodes) / 20) == 0:
                logging.debug("[%d] (%.2f %%) -- tokens = %d",
                              nodes_processed,
                              nodes_processed / len(sorted_nodes) * 100,
                              len(node_tokens))

        raise InputError("Could not reach the final node of word lattice.")

    def _propagate(self, tokens, link, lm_scale, wi_penalty):
        Propagates tokens to given link or to end of sentence.

Instances