3146d1478501f47934e3a1136f731ba9e948420c,snips_nlu/resources.py,,_load_gazetteers,#Any#Any#,185

Before Change

                normalized = normalize(line.strip())
                if normalized:
                    token_values = (t.value
                                    for t in tokenize(normalized, language))
                    normalized = get_default_sep(language).join(token_values)
                    gazetteers[gazetteer_name].add(normalized)
    return gazetteers

After Change

    for gazetteer_name in gazetteer_names:
        gazetteer_path = (gazetteers_dir / gazetteer_name).with_suffix(".txt")
        with gazetteer_path.open(encoding="utf8") as f:
            gazetteers[gazetteer_name] = set(v.strip() for v in f)
    return gazetteers

Instances