a3e9a51bb758111edf59f7b6dbcc5205d0f0364b,tensorflow_datasets/core/download/downloader.py,_Downloader,_sync_download,#_Downloader#Any#Any#,80

Before Change

    Synchronous version of `download` method.
    checksum = self._checksumer()
    try:
      with contextlib.closing(urllib.request.urlopen(url)) as response:
        fname = util.get_file_name(response.geturl())
        path = os.path.join(destination_path, fname)
        _copy_response_to_file(response, path, checksum)
    except urllib.error.HTTPError as err:
      raise HTTPError(url, err.code, err.reason)
    return checksum.hexdigest()

After Change

  def _sync_download(self, url, destination_path):
    Synchronous version of `download` method.
    checksum = self._checksumer()
    session = requests.Session()
    if _DRIVE_URL.match(url):
      url = self._get_drive_url(url, session)
    response = session.get(url, stream=True)
    fname = util.get_file_name(response.url)
    path = os.path.join(destination_path, fname)
    with gfile.Open(path, "wb") as file_:
      for block in response.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE):
        checksum.update(block)
        // TODO(pierrot): Test this is faster than doing checksum in the end

Instances