d42032d825201b5eee463fbaa7fb15ac46ed0643,opennmt/tokenizers/opennmt_tokenizer.py,,create_tokenizer,#Any#,12

Before Change

  _set(kwargs, "segment_alphabet_change")
  _set(kwargs, "segment_alphabet")

  return pyonmttok.Tokenizer(config.get("mode", "conservative"), **kwargs)


class OpenNMTTokenizer(Tokenizer):

After Change

  
  kwargs = copy.deepcopy(config)
  mode = "conservative"
  if "mode" in kwargs:
    mode = kwargs["mode"]
    del kwargs["mode"]
  return pyonmttok.Tokenizer(mode, **kwargs)


class OpenNMTTokenizer(Tokenizer):

Instances