8c8417497aff9741449d9bd3681cad6aea06283a,tmtoolkit/preprocess/_tmpreproc.py,TMPreproc,get_dtm,#TMPreproc#Any#Any#,947

Before Change

            logger.info("generating DTM")

            workers_res = self._get_results_seq_from_workers("get_num_unique_tokens_per_doc")
            dtm_alloc_size = sum(flatten_list([list(num_unique_per_doc.values())
                                               for num_unique_per_doc in workers_res]))
            vocab = self.get_vocabulary(sort=True)

            self._cur_dtm = create_sparse_dtm(vocab, self.doc_labels, self.tokens, dtm_alloc_size, dtype=dtype,
                                              vocab_is_sorted=True)

After Change

                // sort according to document labels
                // dtm_vocab == self.vocabulary (both sorted) but dtm_doc_labels is not sorted
                self._cur_dtm = dtm[np.argsort(dtm_doc_labels), :]
                vocab = dtm_vocab.tolist()
            else:
                self._cur_dtm = csr_matrix((0, 0), dtype=dtype)  // empty sparse matrix
                vocab = list()
        else:

Instances