3052e961e54ead28c717b6bbf5c785946e727fa9,finetune/input_pipeline.py,BasePipeline,get_target_input_fn,#BasePipeline#Any#Any#,316

Before Change

        batch_size = batch_size or self.config.batch_size
        TS = tf.TensorShape
        shapes ={
                    "features": TS([768,]),
                    "sequence_features": TS([self.config.max_length,768]),
                    "attention_weights": TS([12,512,512])
                }

        if self.config.base_model in [GPTModel, GPTModelSmall]:
            output_types = {"features":tf.float32, "sequence_features":tf.float32, "attention_weights":tf.float32}
        else:
            output_types = (tf.float32, tf.float32)
        
        tf_dataset = lambda: tf.data.Dataset.from_generator(dataset_encoded,output_types=output_types).batch(batch_size)
        //tf_dataset = lambda: tf.data.Dataset.from_tensor_slices(features).batch(batch_size)
        return tf_dataset
                                                        

After Change

    def get_target_input_fn(self, features, batch_size=None):
        batch_size = min(len(features),64)
        features = pd.DataFrame(features).to_dict("list")
        for key in features:
            features[key] = np.array(features[key])
        tf_dataset = lambda: tf.data.Dataset.from_tensor_slices(dict(features)).batch(batch_size)
        return tf_dataset
            
    @property

Instances