d5b67dee6fe4630c79424deb87f587f616eba02d,src/fetch_yelp_info.py,,,#,84

Before Change

companies_w_meal_expense = companies()
fetched_companies = load_companies_dataset()
companies_to_fetch = remaining_companies(fetched_companies, companies_w_meal_expense)

for i, company in companies_to_fetch.iterrows():
    print("%s: Fetching %s - City: %s" % (i, company["trade_name"], company["city"]))

    full_address = "{}, {}, {}".format(company["neighborhood"], company["city"], company["state"])
    fetched_company = fetch_yelp_info(term=company["trade_name"], latitude=company["latitude"], longitude=company["longitude"], radius=10000)

    if fetched_company:
        print("Successfuly matched %s" % fetched_company["name"])
        normalized = json_normalize(fetched_company)
        normalized["scraped_at"] = datetime.datetime.utcnow().isoformat()
        normalized["trade_name"] = company["trade_name"]
        normalized["cnpj"] = company["cnpj"]
        normalized["clean_cnpj"] = company["clean_cnpj"]
        fetched_companies = pd.concat([fetched_companies, normalized])

After Change

  companies_w_meal_expense = companies()
  fetched_companies = load_companies_dataset()
  COMPANIES_DATASET_PATH = os.path.join("data", "2016-09-03-companies.xz")
  companies_to_fetch = remaining_companies(fetched_companies, companies_w_meal_expense).reset_index()

  for index, company in companies_to_fetch.iterrows():
    print("%s: Fetching %s - City: %s" % (index, company["trade_name"], company["city"]))

    full_address = "{}, {}, {}".format(company["neighborhood"], company["city"], company["state"])
    fetched_company = fetch_yelp_info(term=company["trade_name"], latitude=company["latitude"], longitude=company["longitude"], radius=10000)

    if fetched_company:
      print("Successfuly matched %s" % fetched_company["name"])
      normalized = json_normalize(fetched_company)
      normalized["scraped_at"] = datetime.datetime.utcnow().isoformat()
      normalized["trade_name"] = company["trade_name"]
      normalized["cnpj"] = company["cnpj"]
      fetched_companies = pd.concat([fetched_companies, normalized])
    else:

Instances