bb1b1c6e27b273a66135d02a04232835d5bfc3ca,allennlp/data/tokenizers/pretrained_transformer_tokenizer.py,PretrainedTransformerTokenizer,tokenize,#PretrainedTransformerTokenizer#Any#,46

Before Change

    def tokenize(self, text: str) -> List[Token]:
        // TODO(mattg): track character offsets.  Might be too challenging to do it here, given that
        // ``transformers``` is dealing with the whitespace...
        token_strings = self._start_tokens + self._tokenizer.tokenize(text) + self._end_tokens
        return [Token(t) for t in token_strings]


def _guess_start_and_end_token_defaults(model_name: str) -> Tuple[List[str], List[str]]:
    if "bert" in model_name:

After Change

        This method only handels a single sentence (or sequence) of text.
        Refer to the ``tokenize_sentence_pair`` method if you have a sentence pair.
        
        return self._tokenize(text)

Instances