2c5f22199b46837b9801378ce172dcd13f125769,gluonnlp/vocab/subwords.py,NGramHashes,_word_to_hashes,#NGramHashes#Any#,211

Before Change

            if word not in self.special_tokens:
                hashes = nd.array([
                    self.fasttext_hash_asbytes((u"<" + word + u">")[i:i + N]) % self.num_subwords
                    for N in self.ngrams for i in range((len(word) + 2) - N + 1)
                ])
            else:
                hashes = nd.zeros(shape=0)

After Change

    def _word_to_hashes(self, word):
        if word not in self.special_tokens:
            word_enc = bytearray((u"<" + word + u">").encode("utf-8"))
            hashes = _fasttext_ngram_hashes(
                memoryview(word_enc), ns=self._ngrams,
                bucket_size=self.num_subwords)
        else:
            hashes = []
        return hashes

Instances