70a99a9759c8fb8d5731270639ebf95dd0e02a04,preprocess.py,,build_save_in_shards_using_shards_size,#Any#Any#Any#Any#Any#,49

Before Change

            src_data = fsrc.readlines()
            tgt_data = ftgt.readlines()

            src_corpus = "".join(src_corpus.split(".")[:-1])
            tgt_corpus = "".join(tgt_corpus.split(".")[:-1])

            num_shards = int(len(src_data) / opt.shard_size)
            for x in range(num_shards):

After Change

        torch.save(dataset, pt_file)

        ret_list.append(pt_file)
        os.remove(src)
        os.remove(tgt_list[index])
        del dataset.examples
        gc.collect()
        del dataset

Instances