9def69a805132ff7549744b5b30a2fa7531fa405,tmtoolkit/preprocess/_tmpreproc.py,TMPreproc,load_tokens_dataframe,#TMPreproc#Any#,263

Before Change

        tokens = {}
        for dl, doc_df in tokendf.groupby(level=0):
            doc_df = doc_df.reset_index()
            doc_df = doc_df.loc[:, doc_df.columns.difference(ind_names)]
            tokens[dl] = doc_df

        return self.load_tokens(tokens)

After Change

        // convert big dataframe to dict of document token dicts to be used in load_tokens
        tokens = {}
        for dl in dt.unique(tokendf[:, dt.f.doc]).to_list()[0]:
            doc_df = tokendf[dt.f.doc == dl, :]
            colnames = list(doc_df.names)
            colnames.pop(colnames.index("doc"))

Instances