5818901a8b1b2301d7264dc7de58730bc008e5ef,petastorm/pytorch.py,BatchedDataLoader,_iter_impl,#BatchedDataLoader#,298

Before Change

        // the requested batch_size ready.

        keys = None
        if self.shuffling_queue_capacity > 0:
            // We can not know what is the reasonable number to use for the extra capacity, so we set a huge number
            // and give up on the unbound growth protection mechanism.
            // To keep the same behavior as DataLoader, we need to increase the shuffling_queue_capacity
            min_after_dequeue = self.shuffling_queue_capacity - 1
            shuffling_queue_capacity = min_after_dequeue + self.batch_size
            self._shuffling_buffer = BatchedRandomShufflingBuffer(
                shuffling_queue_capacity,
                min_after_retrieve=min_after_dequeue,
                extra_capacity=100000000,
                batch_size=self.batch_size
            )
        else:
            self._shuffling_buffer = BatchedNoopShufflingBuffer(batch_size=self.batch_size)

        for row in self.reader:
            // Default collate does not work nicely on namedtuples and treat them as lists
            // Using dict will result in the yielded structures being dicts as well
            row_as_dict = row._asdict()

After Change

            other_shuffling_buffer.add_many(batch.values())
            yield batch

        if self.inmemory_cache_all:
            for epoch in range(self.num_epochs - 1 if self.num_epochs else sys.maxsize):
                other_shuffling_buffer.finish()
                self._shuffling_buffer = other_shuffling_buffer
                other_shuffling_buffer = instantiate_buffer_fn()
                for batch in self._yield_batches(keys):
                    if self.num_epochs is None or self.num_epochs and epoch != self.num_epochs-2:
                        // We skip populating the other buffer in the last epoch.
                        other_shuffling_buffer.add_many(batch.values())
                    yield batch

    def _yield_batches(self, keys):
        while self._shuffling_buffer.can_retrieve():
            batch = self._shuffling_buffer.retrieve()
            if not isinstance(batch, dict):

Instances