a0351c62548ebddf5c728ec525ad61a4f7827494,tests/hypermodel/test_preprocessor.py,,test_sequence,#,51

Before Change

    texts = ["The cat sat on the mat.",
             "The dog sat on the log.",
             "Dogs and cats living together."]
    tokenize = preprocessor.TextToIntSequence()
    dataset = tf.data.Dataset.from_tensor_slices(texts)
    tokenize.set_hp(kerastuner.HyperParameters())
    for x in dataset:
        tokenize.update(x)
    tokenize.finalize()
    tokenize.set_config(tokenize.get_config())

    weights = tokenize.get_weights()
    tokenize.clear_weights()
    tokenize.set_weights(weights)

    for a in dataset:
        tokenize.transform(a)

    def map_func(x):
        return tf.py_function(tokenize.transform,
                              inp=[x],
                              Tout=(tf.int64,))

    new_dataset = dataset.map(map_func)
    for _ in new_dataset:
        pass
    assert isinstance(new_dataset, tf.data.Dataset)


def test_ngram():

After Change

    texts = ["The cat sat on the mat.",
             "The dog sat on the log.",
             "Dogs and cats living together."]
    dataset = tf.data.Dataset.from_tensor_slices(texts)
    new_dataset = run_preprocessor(preprocessor.TextToIntSequence(),
                                   dataset,
                                   common.generate_data(dtype="dataset"),
                                   tf.int64)
    assert isinstance(new_dataset, tf.data.Dataset)


def test_ngram():

Instances