99d0fb1a8a00c80c97271b2cc58d52c4fcff0032,jieba/__init__.py,Tokenizer,load_userdict,#Tokenizer#Any#,347
Before Change
line = ln.strip().decode("utf-8").lstrip("\ufeff")
if not line:
continue
tup = line.split(" ")
freq, tag = None, None
if len(tup) == 2:
if tup[1].isdigit():
freq = tup[1]
else:
tag = tup[1]
elif len(tup) > 2:
freq, tag = tup[1], tup[2]
self.add_word(tup[0], freq, tag)
except Exception:
raise ValueError(
"invalid dictionary entry in %s at Line %s: %s" % (
After Change
line = ln.strip()
if not isinstance(f, text_type):
try:
line = line.decode("utf-8").lstrip("\ufeff")
except UnicodeDecodeError:
raise ValueError("dictionary file %s must be utf-8" % f.name)
if not line:
continue
// match won"t be None because there"s at least one character
word, freq, tag = re_userdict.match(line).groups()
if freq is not None:
freq = freq.strip()
if tag is not None:
tag = tag.strip()
self.add_word(word, freq, tag)
def add_word(self, word, freq=None, tag=None):
In pattern: SUPERPATTERN
Frequency: 3
Non-data size: 16
Instances
Project Name: fxsjy/jieba
Commit Name: 99d0fb1a8a00c80c97271b2cc58d52c4fcff0032
Time: 2015-11-09
Author: abcdoyle888@gmail.com
File Name: jieba/__init__.py
Class Name: Tokenizer
Method Name: load_userdict
Project Name: hellohaptik/chatbot_ner
Commit Name: e870ef14c590502fb0dc5ff3199e2602a87ec008
Time: 2019-03-18
Author: jain.chirag925@gmail.com
File Name: ner_v1/detectors/numeral/budget/budget_detection.py
Class Name: BudgetDetector
Method Name: _detect_max_budget
Project Name: hellohaptik/chatbot_ner
Commit Name: e870ef14c590502fb0dc5ff3199e2602a87ec008
Time: 2019-03-18
Author: jain.chirag925@gmail.com
File Name: ner_v1/detectors/numeral/budget/budget_detection.py
Class Name: BudgetDetector
Method Name: _detect_min_budget