b156eba3ba09229a10692cffdc26bb8604b9f651,api-examples/transformer_utils.py,MultiFileLoader,__iter__,#MultiFileLoader#,410

Before Change

        self.vectorizer.mxlen = self.nctx

        for file in files[start_idx:end_idx]:
            with open(file) as rf:
                for line in rf:
                    response = self.process_line(line)

After Change

        for file_idx in read_file_order:
            file = files[file_idx]
            with open(file) as rf:
                lines = rf.readlines()
                if self.shuffle:
                    random.shuffle(lines)
                for l in lines:
                    response = self.process_line(l)

Instances