78962a65d919e3f21ecdf1d58155efd0c4c3f815,finetune/base_models/bert/encoder.py,BERTEncoder,_encode,#BERTEncoder#Any#Any#,44

Before Change

            subtokens = self.tokenizer.tokenize(text)
            subtoken_locs = [0]
            for tok in subtokens:
                if tok.startswith("////"):
                    subtoken_locs.append(subtoken_locs[-1] + len(tok) - 2)
                else:
                    subtoken_locs.append(subtoken_locs[-1] + len(tok) + 1)
            subtoken_locs = subtoken_locs[1:]

            batch_tokens.append(subtokens)
            batch_token_idxs.append(self.tokenizer.convert_tokens_to_ids(subtokens))

After Change

                label = labels[i]

            subtokens, token_idxs = self.tokenizer.tokenize(text)
            subtoken_locs = [l[1] for l in token_idxs]

            batch_tokens.append(subtokens)
            batch_token_idxs.append(self.tokenizer.convert_tokens_to_ids(subtokens))
            batch_character_locs.append(subtoken_locs)

Instances