52d69d629e9083fdd4e5ff14e2c52d727bfd9335,nala/preprocessing/tokenizers.py,NLTKTokenizer,tokenize,#NLTKTokenizer#Any#,26

Before Change

        :type dataset: nala.structures.data.Dataset
        
        for part in dataset.parts():
            part.sentences = [[Token(word) for word in word_tokenize(sentence)] for sentence in part.sentences]


class TmVarTokenizer(Tokenizer):
    

After Change

        for part in dataset.parts():
            so_far = 0
            for index, sentence in enumerate(part.sentences):
                part.sentences[index] = []
                for token_word in word_tokenize(sentence):
                    token_start = part.text.find(token_word, so_far)
                    so_far = token_start + len(token_word)
                    part.sentences[index].append(Token(token_word, token_start))

Instances