0f4616ac9a4f4db956a4e73c70218ad113af5e68,tensorflow_datasets/scripts/cleanup/url_filename_recorder.py,,main,#Any#,61

Before Change

    for url_path in _collect_path_to_update():
      url_infos = checksums.load_url_infos(url_path)
      cached_url_infos[url_path] = url_infos
      future_filenames[url_path] = executor.map(_request_filename, url_infos)
    for path, future_filename in future_filenames.items():
      old_url_infos = cached_url_infos[path]
      updated_url_infos = _update_url_infos(old_url_infos, future_filename)
      checksums.save_url_infos(path, updated_url_infos)

After Change

  // Remove duplicate urls
  all_url_infos = {}
  for url_infos in path_to_url_infos.values():
    all_url_infos.update(url_infos)

  with futures.ThreadPoolExecutor(max_workers=100) as executor:

Instances