f688c3c4a8c254bf5cdefd4e2e61587c0ef7a46e,src/preprocess.py,,build_tasks,#Any#,63

Before Change

    // 4) Index tasks using vocab, using previous preprocessing if available.
    preproc_file = os.path.join(args.exp_dir, args.preproc_file)
    if os.path.exists(preproc_file) and not args.reload_vocab and not args.reload_indexing:
        preproc = pkl.load(open(preproc_file, "rb"))
        save_preproc = 0
    else:
        preproc = {}
    for task in tasks:
        if task.name in preproc:
            train, val, test = preproc[task.name]
            task.train_data = train

After Change

    if not os.path.isdir(preproc_dir):
        os.mkdir(preproc_dir)
    preproc_file_names = []
    if not args.reload_vocab and not args.reload_indexing:
        for file in os.listdir(preproc_dir):
            preproc_file_names.append(file)
    for task in tasks:
        if task.name in preproc_files:
            train, val, test = get_task_generator(task.name)
            task.train_data = train

Instances