ba0de016423c1ef3370cf79a51aa7a5b8b6f03bb,petastorm/tests/test_ngram_end_to_end.py,NgramEndToEndDatasetToolkitTest,_test_continuous_ngram_tf,#NgramEndToEndDatasetToolkitTest#Any#,89

Before Change

            ids = range(99)
            dataset_dicts = create_test_dataset(tmp_url, ids, num_files=1)
            ngram = NGram(fields=ngram_fields, delta_threshold=10, timestamp_field=TestSchema.id)
            reader = Reader(
                schema_fields=ngram,
                dataset_url=tmp_url,
                reader_pool=ThreadPool(1),
                shuffle_options=ShuffleOptions(False))

            readout_examples = tf_tensors(reader)

            // Make sure we have static shape info for all fields

After Change

    

    ngram = NGram(fields=ngram_fields, delta_threshold=10, timestamp_field=TestSchema.id)
    with Reader(
            schema_fields=ngram,
            dataset_url=dataset_num_files_1.url,
            reader_pool=ThreadPool(1),
            shuffle_options=ShuffleOptions(False)) as reader:

        readout_examples = tf_tensors(reader)

        // Make sure we have static shape info for all fields
        for timestep in readout_examples:
            for field in readout_examples[timestep]:
                assert field.get_shape().dims is not None

        // Read a bunch of entries from the dataset and compare the data to reference
        expected_id = 0
        with tf.Session() as sess:
            for _ in range(5):
                actual = sess.run(readout_examples)

Instances