49d41416e440926f0a9a8243b4d77f6f5468efe9,app/server/utils.py,CoNLLHandler,parse,#CoNLLHandler#Any#,100

Before Change

    def parse(self, file):
        words, tags = [], []
        for i, line in enumerate(file, start=1):
            line = line.decode("utf-8")
            line = line.strip()
            if line:
                try:
                    word, tag = line.split("\t")

After Change

class JSONParser(FileParser):

    def parse(self, file):
        data = []
        for i, line in enumerate(file, start=1):
            if len(data) >= IMPORT_BATCH_SIZE:
                yield data
                data = []
            try:
                j = json.loads(line)
                j["meta"] = json.dumps(j.get("meta", {}))

Instances