f59d5873a23dfa3864d3ff5e3ed1070a3ca762ec,finetune/input_pipeline.py,BasePipeline,_dataset_with_targets,#BasePipeline#Any#Any#Any#,127

Before Change

        dataset_encoded = lambda: itertools.chain.from_iterable(
            map(lambda xy: self.text_to_tokens_mask(*xy), dataset()))
        shape_def = self.feed_shape_type_def()
        if not callable(Y) and self.config.chunk_long_sequences and train:
            dataset_encoded_list = list(dataset_encoded())  // come up with a more principled way to do this .
            self.config.dataset_size = len(dataset_encoded_list)
        return Dataset.from_generator(lambda: self.wrap_tqdm(dataset_encoded(), train), *shape_def)

    def _dataset_without_targets(self, Xs, train):
        if not callable(Xs):

After Change

        shape_def = self.feed_shape_type_def()
        if not callable(Y) and train:
            dataset_encoded_list = list(dataset_encoded())
            counter = Counter()
            for doc, target_arr in dataset_encoded_list:
                counter.update(target_arr[doc["mask"].astype(np.bool)])
            class_counts = {
                self.label_encoder.inverse_transform([target])[0]: count 
                for target, count in counter.items()
            }
            self.config.class_weights = compute_class_weights(self.config.class_weights, class_counts=class_counts)
            self.config.dataset_size = len(dataset_encoded_list)
        return Dataset.from_generator(lambda: self.wrap_tqdm(dataset_encoded(), train), *shape_def)

Instances