038515ccfb6df094e22b9f14fcf9d8c391bc4eaa,torchtext/data/utils.py,,ngrams_iterator,#Any#Any#,117

Before Change

        >>> ["here", "here we", "we", "we are", "are"]
    

    for i in range(0, len(token_list)):
        x = token_list[i]
        yield x
        for j in range(i + 1, min(i + ngrams, len(token_list))):
            x += " " + token_list[j]
            yield x


class RandomShuffler(object):
    Use random functions while keeping track of the random state to make it
    reproducible and deterministic.

After Change

    for x in token_list:
        yield x
    for n in range(2, ngrams + 1):
        for x in _get_ngrams(n):
            yield " ".join(x)


class RandomShuffler(object):
    Use random functions while keeping track of the random state to make it
    reproducible and deterministic.

Instances