c177cdcf25c2292c6c1ff5679145da6dbebfa407,perfkitbenchmarker/scripts/spark_sql_runner.py,,main,#Any#,46

Before Change

  results = []
  for script in args.sql_scripts:
    // Read script from object storage using rdd API
    query = "\n".join(spark.sparkContext.textFile(script).collect())

    try:
      logging.info("Running %s", script)
      start = time.time()

After Change

           .getOrCreate())
  table_metadata = []
  if args.table_metadata:
    table_metadata = json.loads(load_file(spark, args.table_metadata)).items()
  for name, (fmt, options) in table_metadata:
    logging.info("Loading %s", name)
    spark.read.format(fmt).options(**options).load().createTempView(name)

Instances