a0d11e695f5d2c3773332e866eba5ad64944b79e,tmtoolkit/preprocess/_tmpreproc.py,TMPreproc,get_dtm,#TMPreproc#,628

Before Change

    def get_dtm(self):
        logger.info("generating DTM")

        vocab, doc_labels, docs_terms, dtm_alloc_size = get_vocab_and_terms(self.tokens)
        dtm = create_sparse_dtm(vocab, doc_labels, docs_terms, dtm_alloc_size)

        return doc_labels, vocab, dtm

    @property

After Change

            return self._cur_dtm

        workers_res = self._get_results_seq_from_workers("get_num_unique_tokens_per_doc")
        dtm_alloc_size = sum(flatten_list([list(num_unique_per_doc.values()) for num_unique_per_doc in workers_res]))

        self._cur_dtm = create_sparse_dtm(self.vocabulary, self.doc_labels, self.tokens, dtm_alloc_size)

        return self._cur_dtm

    @property

Instances