383bdf3b22faa2be460caf871e58130ec90ecb1a,delft/sequenceLabelling/preprocess.py,,to_vector_bert,#Any#Any#Any#Any#Any#Any#,294

Before Change

    vectors based on the BERT contextualized embeddings, introducing
    padding token when appropriate
    
    subtokens = []
    for i in range(0, len(tokens)):
        local_tokens = []
        for j in range(0, min(len(tokens[i]), maxlen)):
            if lowercase:
                local_tokens.append(_lower(tokens[i][j]))
            else:
                local_tokens.append(tokens[i][j])
        if extend:
            local_tokens.append(UNK)
        subtokens.append(local_tokens)
    vector = embeddings.get_sentence_vector_only_BERT(subtokens)
    return vector

After Change

    vectors based on the BERT contextualized embeddings, introducing
    padding token when appropriate
    
    subtokens = get_subtokens(tokens, maxlen, extend, lowercase)
    vector = embeddings.get_sentence_vector_only_BERT(subtokens)
    return vector

Instances