7fa458e2c4c5df5a9d2cc4e66b2472cd9f3377a8,python/baseline/model.py,Tagger,predict_text,#Tagger#Any#,207

Before Change

        //lengths = zero_alloc(1, dtype=int)
        //lengths[0] = min(len(tokens), mxlen)

        data = featurizer.run(tokens)
        lengths = data["lengths"]
        indices = self.predict(data)[0]
        output = []
        for j in range(lengths[0]):

After Change

            mxlen = kwargs.get("mxlen", self.mxlen if hasattr(self, "mxlen") else len(tokens))
            maxw = kwargs.get("maxw", self.maxw if hasattr(self, "maxw") else max([len(token) for token in tokens]))
            word_tokenizer = Dict1DVectorizer(mxlen=mxlen, fields="text")
            char_tokenizer = Dict2DVectorizer(mxlen=mxlen, mxwlen=maxw, fields="text")
            vectorizers = {"word": word_tokenizer, "char": char_tokenizer}

        // This might be inefficient if the label space is large

        label_vocab = revlut(self.get_labels())
        batch_dict = dict()
        for k, vectorizer in vectorizers.items():
            value, length = vectorizer.run(tokens, self.embeddings[k].vocab)
            batch_dict[k] = value
            if length is not None:
                batch_dict["{}_lengths".format(k)] = length

        indices = self.predict(batch_dict)[0]
        output = []
        for j in len(tokens):
            output.append((tokens[j], label_vocab[indices[j].item()]))

Instances