b0c57631bfc686c9f5572681750c305cfc9754af,snippets/ch05/ner.py,,,#,5

Before Change

import nltk
from nltk.corpus.reader.plaintext import CategorizedPlaintextCorpusReader

path = os.path.join(os.getcwd(), "debates")

DOC_PATTERN = r"(?!\.)[\w_\s]+/[\w\s\d\-]+\.txt"
CAT_PATTERN = r"([\w_\s]+)/.*"

corpus = CategorizedPlaintextCorpusReader(
    path, DOC_PATTERN, cat_pattern=CAT_PATTERN)


def tag_corpus(corpus):
    return [nltk.pos_tag(sent) for sent in corpus.sents()]

After Change

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

GOODLABELS = frozenset(["PERSON", "ORGANIZATION", "FACILITY", "GPE", "GSP"])
// GPE is Geo-Political Entity, GSP is Geo-Socio-Political group

Instances