63bde2f7cf28dbf6bca6e77fe0b0a9966dc6aee6,finetune/encoding/input_encoder.py,,tokenize_context,#Any#Any#Any#,203

Before Change

                current_char_loc += 1
                if current_char_loc >= len(context_by_char_loc):
                    // TODO: this is a workaround that has no guarantees of being correct
                    raise ValueError("Context cannot be fully matched as it appears to not cover the end of the sequence")
            tokenized_context.append(context_by_char_loc[current_char_loc][1])

    assert len(tokenized_context) == len(encoded_output.token_ends)
    // padded value doesn"t matter since it will be masked out

After Change

                if current_char_loc >= len(context_by_char_loc):
                    // TODO: this is a workaround that has no guarantees of being correct
                    raise ValueError("Context cannot be fully matched as it appears to not cover the end of the sequence for token {}".format(token))
            if token.strip() not in context_by_char_loc[current_char_loc][2]:
                warnings.warn("subtoken: {} has matched up with the context for token: {}".format(repr(token), repr(context_by_char_loc[current_char_loc][2])))
            tokenized_context.append(context_by_char_loc[current_char_loc][1])

    assert len(tokenized_context) == len(encoded_output.token_ends)
    // padded value doesn"t matter since it will be masked out

Instances