3e2ebf76a530eca5305d1feba6be9c531ee33d71,gensim/corpora/hashdictionary.py,HashDictionary,filter_extremes,#HashDictionary#Any#Any#Any#,172

Before Change

        no_above_abs = int(no_above * self.num_docs) // convert fractional threshold to absolute threshold

        // statistics of which ids do we keep?
        good_ids = (hash_id for hash_id in self.keys() if no_below <= self.dfs.get(hash_id, 0) <= no_above_abs)

        if keep_n is not None:
            good_ids = sorted(good_ids, key=lambda item: self.dfs.get(item, 0), reverse=True)
            good_ids = good_ids[:keep_n]
        good_ids = set(good_ids)

        self.id2token = dict((tokenid, freq) for tokenid, freq in self.id2token.iteritems() if tokenid in good_ids)

After Change

        
        no_above_abs = int(no_above * self.num_docs) // convert fractional threshold to absolute threshold

        self.dfs_debug = dict((word, freq) for word, freq in self.dfs_debug.iteritems() if no_below <= freq <= no_above_abs)
        self.token2id = dict((token, tokenid) for token, tokenid in self.token2id.iteritems() if token in self.dfs_debug)
        self.id2token = dict((tokenid, set(token for token in tokens if token in self.dfs_debug)) for tokenid, tokens in self.id2token.iteritems())
        self.dfs = dict((tokenid, freq) for tokenid, freq in self.dfs.iteritems() if self.id2token.get(tokenid, set()))

Instances