8e11169d5a1a637e0a2c5b1d72408a1d7c7a29a0,setup/prepare_data.py,,prepare,#,29

Before Change

                    vocab_thread2.start()

                    // And thread for saving tokenized data to putput file
                    write_thread = Thread(target=write_lines, args=(out_file, rows))
                    write_thread.start()

        // Last vocab parts and last lines to write
        vocab_thread1 = Thread(target=write_lines, args=(out_file, rows, True))

After Change

                    write_thread = Thread(target=write_lines, args=(out_file, rows))
                    write_thread.start()

                    rows = []

        // Last vocab parts and last lines to write
        vocab_thread1 = Thread(target=append_vocab, args=(rows, 1))
        vocab_thread1.start()

Instances