// Read text
text = Parallel(n_jobs=-1)(delayed(read_text)(str(f)) for f in file_list)
text = Parallel(n_jobs=-1)(delayed(tokenizer.encode)(txt) for txt in text)
// Read text from additional source
for s in read_txt_src:
with open(join(path,s),"r") as txt_file:
After Change
// Read text
text = Parallel(n_jobs=READ_FILE_THREADS)(delayed(read_text)(str(f)) for f in file_list)
//text = Parallel(n_jobs=-1)(delayed(tokenizer.encode)(txt) for txt in text)
text = [tokenizer.encode(txt) for txt in text]
// Read text from additional source
for s in read_txt_src: