756b25abd8925069f7bc06cf5f91b9662737fde0,snips_nlu/slot_filler/feature_factory.py,CustomEntityMatchFactory,_transform,#CustomEntityMatchFactory#Any#,421

Before Change

    def _transform(self, token):
        if self.use_stemming:
            transformed_value = stem_token(token, self.language)
        else:
            transformed_value = normalize_token(token)
        return Token(
            value=transformed_value,
            start=token.start,
            end=token.start + len(transformed_value))

    def build_features(self, builtin_entity_parser=None,
                       custom_entity_parser=None):
        features = []

After Change

    def _transform(self, tokens):
        if self.use_stemming:
            light_tokens = (stem_token(t, self.language) for t in tokens)
        else:
            light_tokens = (normalize_token(t) for t in tokens)
        current_index = 0
        transformed_tokens = []
        for light_token in light_tokens:
            transformed_token = Token(
                value=light_token,
                start=current_index,
                end=current_index + len(light_token))
            transformed_tokens.append(transformed_token)
            current_index = transformed_token.end + 1
        return transformed_tokens

    def build_features(self, builtin_entity_parser=None,
                       custom_entity_parser=None):

Instances