5b8864d1863ac0474db8b657f023341686f4b1fb,benchmark/imdb.py,,imdb_raw,#,9

Before Change

    (x_train, y_train), (x_test, y_test) = imdb.load_data(index_from=index_offset)

    word_to_id = imdb.get_word_index()
    word_to_id = {k: (v + index_offset) for k, v in word_to_id.items()}
    word_to_id["<PAD>"] = 0
    word_to_id["<START>"] = 1
    word_to_id["<UNK>"] = 2

    id_to_word = {value: key for key, value in word_to_id.items()}
    x_train = list(
        map(lambda sentence: " ".join(id_to_word[i] for i in sentence), x_train)
    )
    x_test = list(

After Change

    // set path to dataset
    IMDB_DATADIR = os.path.join(os.path.dirname(dataset), "aclImdb")
    
    classes = ["pos", "neg"]
    train_data = load_files(os.path.join(IMDB_DATADIR, "train"), shuffle=True, categories=classes)
    test_data = load_files(os.path.join(IMDB_DATADIR,  "test"), shuffle=False, categories=classes)

    encoding = "utf-8"
    x_train = np.array(train_data.data)
    y_train = np.array(train_data.target)
    x_test = np.array(test_data.data)
    y_test = np.array(test_data.target)
    return (x_train, y_train), (x_test, y_test)

Instances