99d0fb1a8a00c80c97271b2cc58d52c4fcff0032,jieba/__init__.py,Tokenizer,load_userdict,#Tokenizer#Any#,347

Before Change

                line = ln.strip().decode("utf-8").lstrip("\ufeff")
                if not line:
                    continue
                tup = line.split(" ")
                freq, tag = None, None
                if len(tup) == 2:
                    if tup[1].isdigit():
                        freq = tup[1]
                    else:
                        tag = tup[1]
                elif len(tup) > 2:
                    freq, tag = tup[1], tup[2]
                self.add_word(tup[0], freq, tag)
            except Exception:
                raise ValueError(
                    "invalid dictionary entry in %s at Line %s: %s" % (

After Change

            f = open(f, "rb")
        for lineno, ln in enumerate(f, 1):
            line = ln.strip()
            if not isinstance(f, text_type):
                try:
                    line = line.decode("utf-8").lstrip("\ufeff")
                except UnicodeDecodeError:
                    raise ValueError("dictionary file %s must be utf-8" % f.name)
            if not line:
                continue
            // match won"t be None because there"s at least one character
            word, freq, tag = re_userdict.match(line).groups()

Instances