c8a46a31f394ba2d0aef3993fdd49e355afbb5af,finetune/input_pipeline.py,BasePipeline,_dataset_with_targets,#BasePipeline#Any#Any#Any#Any#,148

Before Change

    def _dataset_with_targets(self, Xs, Y, train, context=None):
        if context is None:
            context = [None] * len(Xs)
        if not callable(Xs) and not callable(Y):
            dataset = lambda: zip(Xs, Y, context)
        elif callable(Xs) and callable(Y):
            dataset = lambda: zip(Xs(), Y(), context)

After Change

        return Counter(self.label_encoder.inverse_transform(target_arrs))

    def _dataset_with_targets(self, Xs, Y, train, context=None):
        if context:
            if not callable(Xs) and not callable(Y):
                dataset = lambda: zip(Xs, Y, context)
            elif callable(Xs) and callable(Y):
                dataset = lambda: zip(Xs(), Y(), context)
            else:
                raise ValueError( "Either neither or both of Xs and Y should be callable, not a mixture")

            dataset_encoded = lambda: itertools.chain.from_iterable(
                map(lambda xyc: self.text_to_tokens_mask(*xyc), dataset())
            )
        else:
            if not callable(Xs) and not callable(Y):
                dataset = lambda: zip(Xs, Y)
            elif callable(Xs) and callable(Y):
                dataset = lambda: zip(Xs(), Y())
            else:
                raise ValueError( "Either neither or both of Xs and Y should be callable, not a mixture")
            dataset_encoded = lambda: itertools.chain.from_iterable(
                map(lambda xy: self.text_to_tokens_mask(*xy), dataset())
            )

        if not callable(Y) and train:
            dataset_encoded_list = list(dataset_encoded())
            class_counts = self._compute_class_counts(dataset_encoded_list)
            self.config.dataset_size = len(dataset_encoded_list)

Instances