2c4a6e537126f4123de7c97f30587310d3712c06,allennlp/data/token_indexers/token_characters_indexer.py,TokenCharactersIndexer,token_to_indices,#TokenCharactersIndexer#Any#Any#,45

Before Change

        for character in self._character_tokenizer.tokenize(token)[0]:
            // If our character tokenizer is using byte encoding, the character might already be an
            // int.  In that case, we"ll bypass the vocabulary entirely.
            if isinstance(character, int):
                index = character
            else:
                index = vocabulary.get_token_index(character, self._namespace)

After Change

    @overrides
    def token_to_indices(self, token: Token, vocabulary: Vocabulary) -> List[int]:
        indices = []
        if token.text is None:
            raise ConfigurationError("TokenCharactersIndexer needs a tokenizer that retains text")
        for character in self._character_tokenizer.tokenize(token.text):
            if getattr(character, "text_id", None) is not None:
                // `text_id` being set on the token means that we aren"t using the vocab, we just
                // use this id instead.

Instances