9021ea8b31871e760095086669651dd1f072fce0,gensim/models/fasttext.py,FastTextTrainables,init_ngrams_post_load,#FastTextTrainables#Any#Any#,843

Before Change

        vectors are discarded here to save space.

        
        all_ngrams = []
        wv.vectors = np.zeros((len(wv.vocab), wv.vector_size), dtype=REAL)

        for w, vocab in wv.vocab.items():
            all_ngrams += _compute_ngrams(w, wv.min_n, wv.max_n)
            wv.vectors[vocab.index] += np.array(wv.vectors_ngrams[vocab.index])

        all_ngrams = set(all_ngrams)
        wv.num_ngram_vectors = len(all_ngrams)
        ngram_indices = []
        for i, ngram in enumerate(all_ngrams):
            ngram_hash = _ft_hash(ngram)
            ngram_indices.append(len(wv.vocab) + ngram_hash % self.bucket)
            wv.ngrams[ngram] = i
        wv.vectors_ngrams = wv.vectors_ngrams.take(ngram_indices, axis=0)

        ngram_weights = wv.vectors_ngrams

After Change

        ngram_indices = []
        wv.num_ngram_vectors = 0
        for word in wv.vocab.keys():
            for ngram in _compute_ngrams(word, wv.min_n, wv.max_n):
                ngram_hash = _ft_hash(ngram) % self.bucket
                if ngram_hash in wv.hash2index:
                    continue
                wv.hash2index[ngram_hash] = len(ngram_indices)
                ngram_indices.append(len(wv.vocab) + ngram_hash)
        wv.num_ngram_vectors = len(ngram_indices)
        wv.vectors_ngrams = wv.vectors_ngrams.take(ngram_indices, axis=0)

        ngram_weights = wv.vectors_ngrams

Instances