71432d1890209628e189edf04d7d623160943718,autokeras/hypermodel/preprocessor.py,TextToNgramVector,transform,#TextToNgramVector#Any#Any#,182

Before Change

    def transform(self, x, fit=False):
        sentence = nest.flatten(x)[0].numpy().decode("utf-8")
        data = self.vectorizer.transform([sentence]).toarray()
        if self.selector:
            data = self.selector.transform(data).astype("float32")
        return data[0]

    def output_types(self):
        return (tf.float32,)

After Change

    def transform(self, x, fit=False):
        // Calculate tf at doc level
        tf = np.zeros(len(self.vocabulary), dtype=int)
        x = nest.flatten(x)[0].numpy().decode("utf-8")
        token_pattern = re.compile(r"(?u)\b\w\w+\b")
        tokens = self._word_ngram(token_pattern.findall(x.lower()))

        for feature in tokens:
            if feature in self.vocabulary:
                feature_idx = self.vocabulary[feature]
                tf[feature_idx] += 1
        result = tf * self.k_best_idf_values
        result = normalize([result], norm=self.norm, copy=False)[0]
        return result

    def output_types(self):
        return (tf.float64,)

Instances