1f0509fedfd578d939486d238f1d297f9e7e6279,textblob/classifiers.py,,_get_words_from_dataset,#Any#,45

Before Change

        ``words`` is either a string of a list of tokens.
    """
    tokenizer = WordTokenizer()
    all_words = []
    for words, classification in dataset:
        // Words may either be a string or an iterable
        if isinstance(words, basestring):
            all_words.extend(tokenizer.itokenize(words, include_punc=False))
        else:
            all_words.extend(words)
    return set(all_words)


def basic_extractor(document, train_set):

After Change

            return tokenizer.itokenize(words, include_punc=False)
        else:
            return (w for w in words)
    all_words = chain.from_iterable(tokenize(words) for words, _ in dataset)
    return set(all_words)


def basic_extractor(document, train_set):

Instances