9def69a805132ff7549744b5b30a2fa7531fa405,tmtoolkit/preprocess/_tmpreproc.py,TMPreproc,tokens_dataframe,#TMPreproc#,128

Before Change

        for dl, df in tokens.items():
            df = df.copy()
            df["doc"] = dl
            df["position"] = np.arange(len(df))
            dfs.append(df)

        if dfs:
            res = pd.concat(dfs, ignore_index=True)

After Change

        dfs = []
        for dl, df in tokens.items():
            n = df.shape[0]
            meta_df = dt.Frame({
                "doc": np.repeat(dl, n),
                "position": np.arange(n)
            })

            dfs.append(dt.cbind(meta_df, df))

        if dfs:

Instances