216553203df0f6ab5c9af2b7b46255dbfb69a04c,opennmt/tests/text_test.py,TextTest,testToWordsWithSpacer,#TextTest#Any#Any#,50

Before Change

    expected = tf.constant(expected)
    words = text.tokens_to_words(tokens, subword_token="▁", is_spacer=True)
    words = words.to_tensor()
    words, expected = self.evaluate([words, expected])
    self.assertAllEqual(words, expected)

  def _testPharaohAlignments(self, line, lengths, expected_matrix):
    matrix = text.alignment_matrix_from_pharaoh(

After Change

    [["a", "▁b▁", "c", "d", "▁", "e"], [["a"], ["▁b▁"], ["c", "d"], ["▁", "e"]]],
  ])
  def testToWordsWithSpacer(self, tokens, expected):
    expected = tf.nest.map_structure(tf.compat.as_bytes, expected)
    tokens = tf.constant(tokens)
    words = text.tokens_to_words(tokens, subword_token="▁", is_spacer=True)
    self.assertAllEqual(words.to_list(), expected)

Instances