0efd59ac0f70042e50c01b46c1d4edd463f1728c,tests/encoders/text/test_subword_encoder.py,TestSubwordEncoder,test_eos,#TestSubwordEncoder#Any#,33

Before Change

    def test_eos(self, corpus):
        encoder = SubwordEncoder(corpus, append_eos=True)
        input_ = "This is a sentence"
        encoded = encoder.encode(input_)
        assert encoded[-1] == DEFAULT_EOS_INDEX
        decoded = encoder.decode(encoded)
        assert decoded.find(DEFAULT_EOS_TOKEN) == len(decoded) - len(DEFAULT_EOS_TOKEN)

    def test_sos(self, corpus):
        encoder = SubwordEncoder(corpus, append_sos=True)

After Change

        encoder = SubwordEncoder(corpus, append_eos=True)
        input_ = "This is a sentence"
        assert encoder.encode(input_)[-1] == DEFAULT_EOS_INDEX
        assert encoder.decode(encoder.encode(input_)) == input_

    def test_sos(self, corpus):
        encoder = SubwordEncoder(corpus, append_sos=True)
        input_ = "This is a sentence"

Instances