ae1a7796a3df1add1ee420467afea82bde8bd44c,models/lemma/loader.py,DataLoader,load_file,#DataLoader#Any#Any#,99

Before Change

        with open(filename) as infile:
            while True:
                line = infile.readline()
                if len(line) == 0:
                    break
                line = line.strip()
                if len(line) == 0:
                    if len(cache) > 0:
                        sents.append(cache)
                        cache = []
                else:
                    array = line.split("\t")
                    assert (evaluation and len(array) == 1) or (not evaluation and len(array) == 2)
                    cache += [array]
            if len(cache) > 0:
                sents.append(cache)
        return sents

After Change

        conll_file = conll.CoNLLFile(filename)
        if evaluation:
            data = conll_file.get_words()
            data = [[d] for d in data]
        else:
            data = conll_file.get_words_and_lemmas()
        return conll_file, data

Instances