if context is not None: // each field in texts needs a list of dicts for its context
if type(context) == dict: // here we have one token of context for the text
context_ = [context]
elif type(context[0]) == dict: // this is multiple tokens of context for one example
context_ = context
else: // this is multiple examples, so a list of list of dicts
context_ = context[i]
After Change
// Context is tokenwise, so we need to duplicate contexts for each subtoken of a token, and to match length of labels
if context_ is not None:
original_tokens = []
for char_loc, token in zip(
batch_original_character_locs[i], batch_tokens[i]
):
original_token = 0
for subtoken_idx in range(len(context_)):
if (
