5a68c464caac68e4623f9a7b173bcf24aa719a5c,tensorflow_datasets/text/trivia_qa.py,TriviaQA,_generate_examples,#TriviaQA#Any#Any#Any#,230

Before Change

                  article.get("EntityPages", []), "WikiContext", wiki_dir),
              ["DocSource", "Filename", "Title", "WikiContext"])

          question = article["Question"].strip()
          question_id = article["QuestionId"]
          question_source = article["QuestionSource"].strip()

          yield "%s_%s" % (os.path.basename(filepath), question_id), {
              "entity_pages": entity_pages,
              "search_results": search_results,
              "question": question,
              "question_id": question_id,
              "question_source": question_source,
              "answer": answer_dict,
          }

After Change

    for filepath in files:
      logging.info("generating examples from = %s", filepath)
      fname = os.path.basename(filepath)

      with tf.io.gfile.GFile(filepath) as f:
        current_record = ""
        for line in f:
          if line == "        {\n":
            current_record = line
          elif line.startswith("        }"):  // Handles final record as well.
            article = json.loads(current_record + "}")
            current_record = ""
            example = parse_example(article)
            yield "%s_%s" % (fname, example["question_id"]), example
          else:
            current_record += line

Instances